SLC code not filling all Landsat 7 SR gaps

Question

I am a beginner in Google Earth Engine code and am trying to apply the SLC-gap code to Landsat 7 Surface Reflectance images. Using resources available on StackOverflow, I generated the code below; however, when I bring the images into QGIS, there still appear to be gaps. Is my code incorrect or did I not properly apply it to the images?

First, I masked the clouds based on the pixel_qa band of Landsat SR data:

var cloudMaskL7 = function(image) {
var qa = image.select('pixel_qa');
var cloud = qa.bitwiseAnd(1 << 5)
                  .and(qa.bitwiseAnd(1 << 7))
                  .or(qa.bitwiseAnd(1 << 3));

Then, I removed edge pixels that don't occur in all bands:

var mask2 = image.mask().reduce(ee.Reducer.min());
return image.updateMask(cloud.not()).updateMask(mask2);
 };
var l7 = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR')
              .filterDate('2004-09-15', '2004-12-31')
              .map(cloudMaskL7);

var visParams = {
bands: ['B3', 'B2', 'B1'],
min: 0,
max: 3000,
gamma: 1.4,
};
Map.setCenter(36.197, 31.701,7);
Map.addLayer(l7.median(), visParams);

Then, I mapped the function over one year of Landsat 7 TOA data and took the median and mapped it for Jordan.

var composite = l7.map(cloudMaskL7)
    .median();
Map.setCenter(36.124, 31.663);
Map.addLayer(composite, {bands: ['B4', 'B3', 'B2'], max: 0.3});

Then, I tried to fill the SLC Landsat 7 gaps by applying the USGS L7 Phase-2 Gap filling protocol, using a single kernel size.

var MIN_SCALE = 1/3;
var MAX_SCALE = 3;
var MIN_NEIGHBORS = 144;
var GapFill = function(src, fill, kernelSize) {
var kernel = ee.Kernel.square(kernelSize * 30, 'meters', false);
var common = src.mask().and(fill.mask());
var fc = fill.updateMask(common);
var sc = src.updateMask(common);

Then, I found the primary scaling factors with a regression and interleaved the bands for the regression (assumes the bands have the same names).

   var regress = fc.addBands(sc);
    regress = regress.select(regress.bandNames().sort());
    var fit = regress.reduceNeighborhood(ee.Reducer.linearFit().forEach(src.bandNames()),  kernel, null, false);
    var offset = fit.select('.*_offset');
    var scale = fit.select('.*_scale');

Then, I found the secondary scaling factors using just means and stddev.

 var reducer = ee.Reducer.mean().combine(ee.Reducer.stdDev(), null, true);
    var src_stats = src.reduceNeighborhood(reducer, kernel, null, false);
    var fill_stats = fill.reduceNeighborhood(reducer, kernel, null, false);
    var scale2 = src_stats.select('.*stdDev').divide(fill_stats.select('.*stdDev'));
    var offset2 = src_stats.select('.*mean').subtract(fill_stats.select('.*mean').multiply(scale2));

    var invalid = scale.lt(MIN_SCALE).or(scale.gt(MAX_SCALE));
    scale = scale.where(invalid, scale2);
    offset = offset.where(invalid, offset2);

I applied the scaling and mask off pixels that didn't have enough neighbors.

var count = common.reduceNeighborhood(ee.Reducer.count(), kernel, null, true, 'boxcar');
var scaled = fill.multiply(scale).add(offset)
      .updateMask(count.gte(MIN_NEIGHBORS));

  return src.unmask(scaled, true);
};

var source = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR');
var fill = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR');

I loaded a table of boundaries and filter.

var Jordan = ee.FeatureCollection('USDOS/LSIB_SIMPLE/2017')
    .filter(ee.Filter.or(
        ee.Filter.eq('country_co', 'JO')));
        var clippedJordan = composite.clipToCollection(Jordan);

I displayed the results for Jordan; however, the SLC gaps appear not to be filled. I proceed to calculate the MSAVI2 values using these images, so the remaining gaps influence the results.

var mc = Map.setCenter(36.274, 31.682, 6);
var visParams = {bands: ['B3', 'B2', 'B1']};
Map.addLayer(clippedJordan, visParams, 'clipped composite');

Any advice would be greatly appreciated!

Answer 1

I am not up on the latest Landsat 7 gap-filling technology for SLC-off imagery, but here is a greatly simplified version of what you were trying to do. I removed a lot of (unnecessary?) stuff, increased the kernel size by a lot, and increased the timeframe over which the median replacement is generated. It might get close to what you want:

var cloudMaskL7 = function(image) {
  var qa = image.select('pixel_qa');
  var cloud = qa.bitwiseAnd(1 << 5)
                    .and(qa.bitwiseAnd(1 << 7))
                    .or(qa.bitwiseAnd(1 << 3));
  var mask2 = image.mask().reduce(ee.Reducer.min());
  return image.updateMask(cloud.not()).updateMask(mask2);
};

var l7 = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR')
    .map(cloudMaskL7);

var kernelSize = 10;
var kernel = ee.Kernel.square(kernelSize * 30, 'meters', false);

var GapFill = function(image) {
  var start = image.date().advance(-1, 'year');
  var end = image.date().advance(1, 'year');
  var fill = l7.filterDate(start, end).median();
  var regress = fill.addBands(image); 
  regress = regress.select(regress.bandNames().sort());
  var fit = regress.reduceNeighborhood(ee.Reducer.linearFit().forEach(image.bandNames()), kernel, null, false);
  var offset = fit.select('.*_offset');
  var scale = fit.select('.*_scale');
  var scaled = fill.multiply(scale).add(offset);
  return image.unmask(scaled, true);
};

// TESTING CODE
var point = ee.Geometry.Point(36.124, 31.663);
Map.centerObject(point, 11);

var check = ee.ImageCollection('LANDSAT/LE07/C01/T1_SR')
    .filterBounds(point)
    .filterDate('2004-09-15', '2004-12-31');
var checkImage = ee.Image(check.first());
var visParams = {bands: ['B4', 'B3', 'B2'], min: 200, max: 5500};
Map.addLayer(checkImage, visParams, 'source');

// Test composite.
var checkStart = checkImage.date().advance(-1, 'year');
var checkEnd = checkImage.date().advance(1, 'year');
var composite = l7.filterDate(checkStart, checkEnd).median();
Map.addLayer(composite, visParams, 'median');

// Rough implementation for comparison.
var replaced = checkImage.unmask(composite);
Map.addLayer(replaced, visParams, 'simple');

// Fancy implementation.
var filled = ee.Image(check.map(GapFill).first());
Map.addLayer(filled, visParams, 'filled');

EDIT: The answer now shows how to map this over a collection. Watch out, because I don't know how well this will scale. If you decide to map it over a big area or long time series, you've been warned.