google earth engine - Using cloud confidence to create cloud mask from Landsat 8 BQA?

I want to use the Landsat BQA (TOA images) to create a cloud mask in GEE. In particular I want to mask out all areas where there is high confidence that cloud cover is present. I have tried to adapt the example script, which utilises the single bit cloud band:

var cloudmaskL8 = function(image) {
   var qa = image.select('BQA');  

   var mask = qa.bitwiseAnd(1 << 4).eq(0);
   return image.updateMask(mask);
}

But I do not understand how the 'bitwiseAnd' function works in this context and I'm therefore unable to adapt it for the double bit confidence bands.

How does the 'bitwiseAnd' function work in this context?

Answer

Here is a more flexible approach that can handle dual (or larger) bit patterns. The bit shifts are performed server-side, using the ee.Image.rightShift() and ee.Image.mod() methods.

var RADIX = 2;  // Radix for binary (base 2) data.


var extractQABits = function (qaBand, bitStart, bitEnd) {
  var numBits = bitEnd - bitStart + 1;
  var qaBits = qaBand.rightShift(bitStart).mod(Math.pow(RADIX, numBits));
  return qaBits;
};

Once the bits are extracted, you can test them against a desired pattern using comparison operators (such as ee.Image.lte())

Below is a (verbose) example of masking out clouds and shadows in a Landsat 8 TOA image using its quality band (BQA). Clouds can be identified using the "Cloud Confidence" bits:

Bits 5-6: Cloud Confidence  
  0: Not Determined / Condition does not exist. \

  1: Low, (0-33 percent confidence)
  2: Medium, (34-66 percent confidence)
  3: High, (67-100 percent confidence)

and shadows can be identified using the "Cloud Shadow Confidence" bits:

Bits 7-8: Cloud Shadow Confidence
  0: Not Determined / Condition does not exist.
  1: Low, (0-33 percent confidence)
  2: Medium, (34-66 percent confidence)
  3: High, (67-100 percent confidence)

Here is the full script which displays the final masked image an various intermediate step as map layers:

// Landsat 8 Cloud Masking Example

var RADIX = 2;  // Radix for binary (base 2) data.

// Reference a sample Landsat 8 TOA image.
var image = ee.Image('LANDSAT/LC08/C01/T1_TOA/LC08_043034_20180202');
// Extract the QA band.
var image_qa = image.select('BQA');


var extractQABits = function (qaBand, bitStart, bitEnd) {
  var numBits = bitEnd - bitStart + 1;
  var qaBits = qaBand.rightShift(bitStart).mod(Math.pow(RADIX, numBits));
  //Map.addLayer(qaBits, {min:0, max:(Math.pow(RADIX, numBits)-1)}, 'qaBits');
  return qaBits;
};

// Create a mask for the dual QA bit "Cloud Confidence".
var bitStartCloudConfidence = 5;

var bitEndCloudConfidence = 6;
var qaBitsCloudConfidence = extractQABits(image_qa, bitStartCloudConfidence, bitEndCloudConfidence);
// Test for clouds, based on the Cloud Confidence value.
var testCloudConfidence = qaBitsCloudConfidence.gte(2);

// Create a mask for the dual QA bit "Cloud Shadow Confidence".
var bitStartShadowConfidence = 7;
var bitEndShadowConfidence = 8;
var qaBitsShadowConfidence = extractQABits(image_qa, bitStartShadowConfidence, bitEndShadowConfidence);
// Test for shadows, based on the Cloud Shadow Confidence value.

var testShadowConfidence = qaBitsShadowConfidence.gte(2);

// Calculate a composite mask and apply it to the image.   
var maskComposite = (testCloudConfidence.or(testShadowConfidence)).not();
var imageMasked = image.updateMask(maskComposite);

Map.addLayer(image, {bands:"B4,B3,B2", min:0, max:0.3}, 'original image', false);
Map.addLayer(image.select('BQA'), {min:0, max:10000}, 'BQA', false);
Map.addLayer(testCloudConfidence.mask(testCloudConfidence), {min:0, max:1, palette:'grey,white'}, 'testCloudConfidence');
Map.addLayer(testShadowConfidence.mask(testShadowConfidence), {min:0, max:1, palette:'grey,black'}, 'testShadowConfidence');

Map.addLayer(maskComposite.mask(maskComposite), {min:0, max:1, palette:'green'}, 'maskComposite', false);
Map.addLayer(imageMasked, {bands:"B4,B3,B2", min:0, max:0.3}, 'imageMasked'); 

Script link: https://code.earthengine.google.com/d1de93debe720642c02b1d5b3a57fb82

Depending on your particular use case, you may want to adjust the sensitivity to clouds and shadows by modifying testCloudConfidence and testShadowConfidence.