Hello,
I have a current open topic but since I cannot edit the title and it does not illustrate my issue anymore, I thought of starting a new, better tailored one.
In my script for ProcessingAPI, I build a list of dates (sampleDates) in the updateOutput function using the full scene collection (collections.scenes). Later, in evaluatePixel, I use the samples and scenes arrays to compute daily NDVI and bands.
The problem is that one of the dates present in sampleDates (derived from collections.scenes) does not appear in the scenes passed to evaluatePixel. As a result, when I map over sampleDates to retrieve the NDVI and band values, that extra date is missing—causing my NDVI lookup to return undefined (and falling back to a default value), and in some cases even triggering errors when attempting to access properties of undefined.
I have verified that my key-generation method is consistent (using a helper to produce “YYYY-MM-DD” strings), so I believe the issue stems from the difference between the full scene collection and the filtered scenes in evaluatePixel.
When adding:
"processedSceneDates": scenes.map(scene => getDateKey(scene.date))
to the updateOutputMetadata(), there’s no scene for the date in question: 2018-10-12… but it is there in response[‘userdata.json’][‘acquisitionDates’] so I’m quite surprise at this discrepancy.
Somehow, this date exists in sampleDates, calculated using
collections.scenes.forEach(scene => {
uniqueDailyDates.add(getDateKey(scene.date));
uniqueWeeklyDates.add(getSunday(scene.date));
});
sampleDates = Array.from(uniqueDailyDates).sort();
But doesn’t in scenes object…:
function evaluatePixel(samples, scenes) {
let weeklyNDVI = calcScenesNDVI(samples, scenes);
let dailyNDVI = calcDailyNDVI(samples, scenes);
let dailyBands = calcDailyBands(samples, scenes);
To resume, what I don’t understand so far:
- How the scenes passed to evaluatePixel are determined relative to collections.scenes used in updateOutput?
- Why might there be a date present in the full collection that does not appear in the filtered scenes used for pixel evaluation?
- Any recommended practices for ensuring that my evaluation script only uses dates for which scene data is available in evaluatePixel?
Here’s my full request:
{
“input” : {
“bounds” : {
“properties” : {
“crs” : “http://www.opengis.net/def/crs/EPSG/0/4326”
},
“bbox” : [ -8.510710114131484, 32.427272107731056, -8.509924222820445, 32.428446842115875 ]
},
“data” : [ {
“type” : “sentinel-2-l2a”,
“dataFilter” : {
“timeRange” : {
“from” : “2018-08-23T00:00:00Z”,
“to” : “2018-11-07T00:00:00Z”
},
“maxCloudCoverage” : “90”,
“mosaickingOrder” : “leastCC”
}
} ]
},
“evalscript” : “//VERSION=3\nfunction setup() {\n return {\n input: [{bands: ["B03", "B04", "B08", "SCL", "CLD", "dataMask"], units: "DN"}],\n output: [\n {id: "dailyNDVI", bands: 1, sampleType: "UINT16"},\n {id: "weeklyNDVI", bands: 1, sampleType: "UINT16"},\n {id: "B03", bands: 1, sampleType: "UINT16"},\n {id: "B04", bands: 1, sampleType: "UINT16"},\n {id: "B08", bands: 1, sampleType: "UINT16"}\n ],\n mosaicking: "ORBIT"\n };\n}\n\nconst MIN_INT16 = -32768\nconst OFFSET = 32767\nconst FACTOR = 10000\nlet sampleWeeklyDates = ;\nlet sampleDates = ;\n\nfunction calcNDVI(sample) {\n // Invalid data.\n if (!isValid(sample)) {\n return MIN_INT16\n }\n\n // Cloud detection.\n if (isCloud(sample)) {\n return -2\n }\n\n // Water detection.\n if (isWater(sample)) {\n return 3\n }\n if (isWaterCandidate(sample)) {\n return 2\n }\n\n return index(sample.B08, sample.B04)\n}\n\nfunction isValid(sample) {\n return sample.dataMask === 1\n}\n\nfunction isCloud(sample) {\n // All pixels corresponding to defective (class 1), cloud shadows (3), medium & high cloud probability (8 & 9),\n // thin cirrus (10) or snow (11) in SCL band. List of classes:\n // [https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-2-msi/level-2a/algorithm\n](https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-2-msi/level-2a/algorithm%5Cn) return (sample.SCL === 1 || sample.SCL === 3 || sample.SCL > 7) ? true :\n // Dark area pixels: Mask all pixels whose cloud probability >= 10%.\n (sample.SCL === 2 && sample.CLD >= 10) ? true :\n // Vegetation: Mask all pixels whose cloud probability >= 60%.\n (sample.SCL === 4 && sample.CLD >= 60) ? true :\n // Bare soil: Mask all pixels whose cloud probability >= 5%.\n (sample.SCL === 5 && sample.CLD >= 5) ? true :\n // Unclassified: Mask all pixels whose cloud probability >= 10%.\n (sample.SCL === 7 && sample.CLD >= 10)\n}\n\nfunction isWater(sample) {\n // Mask pixels corresponding to water (class 6) as 3.\n return sample.SCL === 6\n}\n\nfunction isWaterCandidate(sample) {\n // Mask pixels corresponding to dark area (class 2) and unclassified (class 7) to 2 whose cloud probability < 10%\n // and NDWI index (water mask) values > 0.15. Masked values act as a ranking score when we mosaic tiles.\n // Mosaicking takes pixel with max value on overlap, hence we prioritise water pixels (3), then pixels which\n // could be water (2), then vegetation pixels (0-1) and clouds (-2).\n const ndwi = calcNDWI(sample)\n return (sample.SCL === 2 || sample.SCL === 7) && sample.CLD < 10 && ndwi > 0.15\n}\n\nfunction calcNDWI(sample) {\n return index(sample.B03, sample.B08)\n}\n\nfunction getSunday(date) {\n let new_date = new Date(date);\n new_date.setUTCDate(new_date.getUTCDate() + ((7 - new_date.getUTCDay())) % 7);\n return new_date;\n}\n\nfunction getDate(date) {\n let new_date = new Date(date);\n new_date.setUTCDate(new_date.getUTCDate());\n return new_date;\n}\n\n\nfunction calcScenesNDVI(samples, scenes) {\n return samples.reduce(function (acc, sample, index) {\n const sceneWeek = JSON.stringify(getSunday(scenes[index].date));\n const ndvi = calcNDVI(sample);\n\n if (!acc[sceneWeek]) {\n acc[sceneWeek] = MIN_INT16\n }\n\n acc[sceneWeek] = Math.max(acc[sceneWeek], ndvi)\n return acc\n }, {})\n}\n\n\nfunction calcDailyNDVI(samples, scenes) {\n return samples.reduce(function (dailyNdvis, sample, index) {\n const date = JSON.stringify(getDate(scenes[index].date));\n const dailyNdvi = calcNDVI(sample);\n\n if (!dailyNdvis[date]) {\n dailyNdvis[date] = MIN_INT16\n }\n\n dailyNdvis[date] = dailyNdvi\n return dailyNdvis\n }, {})\n}\n\nfunction calcDailyBands(samples, scenes) {\n const res = {};\n for (let i = 0; i < samples.length; i++) {\n const sample = samples[i];\n const scene = scenes[i];\n const date = JSON.stringify(new Date(scene.date));\n res[date] = {\n B03: isValid(sample) ? sample.B03 : 5000,\n B04: isValid(sample) ? sample.B04 : 4000,\n B08: isValid(sample) ? sample.B08 : 5000\n };\n }\n return res;\n}\n\nfunction preProcessScenes (collections) {\n // Sort scenes by dateFrom in ascending order\n collections.scenes.orbits.sort(function (s1, s2) {\n const date1 = new Date(s1.dateFrom)\n const date2 = new Date(s2.dateFrom)\n return date1 - date2\n })\n return collections\n}\n\nfunction updateOutput(outputs, collections) {\n // This function is executed after the setup() and preProcessScenes() functions but before the evaluatePixel().\n // This ensures that the sampleDates array contains all available dates and can be used to ensure consistent order\n // of dates in both the metadata and tiles.\n let uniqueDailyDates = new Set();\n let uniqueWeeklyDates = new Set();\n\n collections.scenes.forEach(scene => {\n uniqueDailyDates.add(JSON.stringify(getDate(scene.date)));\n uniqueWeeklyDates.add(JSON.stringify(getSunday(scene.date)));\n });\n sampleDates = Array.from(uniqueDailyDates).sort();\n sampleWeeklyDates = Array.from(uniqueWeeklyDates).sort();\n\n // Band number must be at least 1 for the request not to fail (even if no tiles were available).\n outputs.weeklyNDVI.bands = Math.max(1, sampleWeeklyDates.length);\n outputs.dailyNDVI.bands = Math.max(1, sampleDates.length);\n outputs.B03.bands = Math.max(1, sampleDates.length);\n outputs.B04.bands = Math.max(1, sampleDates.length);\n outputs.B08.bands = Math.max(1, sampleDates.length);\n}\n\nfunction updateOutputMetadata(scenes, inputMetadata, outputMetadata){\n outputMetadata.userData = {\n "acquisitionDates": […sampleDates],\n "weeklyDates": […sampleWeeklyDates]\n }\n}\n\nfunction evaluatePixel(samples, scenes) {\n let weeklyNDVI = calcScenesNDVI(samples, scenes);\n let dailyNDVI = calcDailyNDVI(samples, scenes);\n let dailyBands = calcDailyBands(samples, scenes);\n\n return {\n dailyNDVI: sampleDates.map(x => dailyNDVI).map(y => y !== undefined ? y * FACTOR + OFFSET : 1000),\n weeklyNDVI: sampleWeeklyDates.map(x => weeklyNDVI).map(y => y !== MIN_INT16 ? y * FACTOR + OFFSET : 0),\n B03: sampleDates.map(date => dailyBands[date] ? dailyBands[date].B03 : 3000),\n B04: sampleDates.map(date => dailyBands[date] ? dailyBands[date].B04 : 3000),\n B08: sampleDates.map(date => dailyBands[date] ? dailyBands[date].B08 : 3000)\n };\n}”,
“output” : {
“responses” : [ {
“identifier” : “dailyNDVI”,
“format” : {
“type” : “image/tiff”
}
}, {
“identifier” : “B03”,
“format” : {
“type” : “image/tiff”
}
}, {
“identifier” : “B04”,
“format” : {
“type” : “image/tiff”
}
}, {
“identifier” : “B08”,
“format” : {
“type” : “image/tiff”
}
}, {
“identifier” : “weeklyNDVI”,
“format” : {
“type” : “image/tiff”
}
}, {
“identifier” : “userdata”,
“format” : {
“type” : “application/json”
}
} ],
“width” : 7,
“height” : 13
}
}
