BatchProcessing + TimeSeries

Documentation
1

Hi everyone, I have an enterprise account, I’ve just started developing using sentinel2 data and python, I’m finding it quite overwhelming.

I’ll go straight to the point:

I need to create a ML model, using input data from 2 countries. I’d like to use all raw bands data from several years (as a time series) and apply post processing locally. What’s the best (and cost-efficient) way to do so?

To go deeper, that’s exactly the data science case explained here:
(Large-scale data preparation — introducing Batch Processing | by Grega Milcinski | Sentinel Hub Blog | Medium)

I’ve already tried a few alternatives, but I don’t get what’s the “right one”:

→ “naive” approach, splitting big area into smaller tiles then looping standard SH requests over every tile and time frame
→ Use Batch Processing, but as it only returns one image per time period, loop it over small time frames.
→ Use multitemporal EvalScript with Batch Processing (though I have almost zero knowledge about JavaScript)
→ Use eolearn library InputTask, that seems to solve the issue, however it’s not optimized using BatchProcess (and tutorials on it are quite outdated)

2

Hi @emidio ,

The most straightforward solution is using multi-temporal evalscript with Batch Processing. This will produce multi-band TIFFs (each band represents data of an acquisition) in AWS S3 bucket . From here you can either download TIFFs to your local machine or directly read TIFFs from the bucket.

To create a multi-temporal evalscript, we have an example script that demonstrates how to request data as a time series in a request. Below is an example evalscript to request raw bands data as a time series that you need. To learn more about evalscript, I suggested going through the Evalscript section of our documentation and the listed tutorials.

//VERSION=3
// define inputs and outputs
function setup() {
  return {
    input: [{
      bands: [
        "B01", "B02", "B03", "B04", "B05", "B06", 
        "B07", "B08", "B8A", "B09", "B11", "B12", 
        "AOT", "SCL", "SNW", "CLD", "CLP", "CLM", 
        "viewZenithMean", "viewAzimuthMean", 
        "sunZenithAngles", "sunAzimuthAngles",
        "dataMask"
      ],
    }],
    output: [
      {id: "b01", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b02", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b03", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b04", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b05", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b06", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b07", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b08", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b8a", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b09", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b11", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "b12", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "aot", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "scl", bands: 1, sampleType: SampleType.UINT8},
      {id: "snw", bands: 1, sampleType: SampleType.UINT8},
      {id: "cld", bands: 1, sampleType: SampleType.UINT8},
      {id: "clp", bands: 1, sampleType: SampleType.UINT8},
      {id: "clm", bands: 1, sampleType: SampleType.UINT8},
      {id: "vzm", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "vam", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "sza", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "saa", bands: 1, sampleType: SampleType.FLOAT32},
      {id: "dm", bands: 1, sampleType: SampleType.UINT8},
    ],
    mosaicking: Mosaicking.ORBIT
  }
}
  
// update output bands to the total number of acquisitions within the input time range
function updateOutput(outputs, collection) {
  const scenes = collection.scenes
  if (scenes.length === 0) {
    n_bands = 1;
  } else {
    n_bands = collection.scenes.length
  }
  Object.values(outputs).forEach((output) => {
    output.bands = n_bands;
  });
}

// write acquisition timestamps to userdata as an output
function updateOutputMetadata(scenes, inputMetadata, outputMetadata) {
  let dates = []
  scenes.forEach(scene => {
    dates.push(scene.date);
  })
  outputMetadata.userData = {
    dates: JSON.stringify(dates)
  };
}
  
// collect values of all acquisitions for each requested band
function evaluatePixel(samples, scenes) {
  let n_observations = samples.length;
  let band_b01 = new Array(n_observations).fill(NaN);
  let band_b02 = new Array(n_observations).fill(NaN);
  let band_b03 = new Array(n_observations).fill(NaN);
  let band_b04 = new Array(n_observations).fill(NaN);
  let band_b05 = new Array(n_observations).fill(NaN);
  let band_b06 = new Array(n_observations).fill(NaN);
  let band_b07 = new Array(n_observations).fill(NaN);
  let band_b08 = new Array(n_observations).fill(NaN);
  let band_b8a = new Array(n_observations).fill(NaN);
  let band_b09 = new Array(n_observations).fill(NaN);
  let band_b11 = new Array(n_observations).fill(NaN);
  let band_b12 = new Array(n_observations).fill(NaN);
  let band_aot = new Array(n_observations).fill(NaN);
  let band_scl = new Array(n_observations).fill(NaN);
  let band_snw = new Array(n_observations).fill(NaN);
  let band_cld = new Array(n_observations).fill(NaN);
  let band_clp = new Array(n_observations).fill(NaN);
  let band_clm = new Array(n_observations).fill(NaN);
  let band_vzm = new Array(n_observations).fill(NaN);
  let band_vam = new Array(n_observations).fill(NaN);
  let band_sza = new Array(n_observations).fill(NaN);
  let band_saa = new Array(n_observations).fill(NaN);
  let band_dm = new Array(n_observations).fill(NaN);

  samples.forEach((sample, index) => {
    band_b01[index] = sample.B01;
    band_b02[index] = sample.B02;
    band_b03[index] = sample.B03;
    band_b04[index] = sample.B04;
    band_b05[index] = sample.B05;
    band_b06[index] = sample.B06;
    band_b07[index] = sample.B07;
    band_b08[index] = sample.B08;
    band_b8a[index] = sample.B8A;
    band_b09[index] = sample.B09;
    band_b11[index] = sample.B11;
    band_b12[index] = sample.B12;
    band_aot[index] = sample.AOT;
    band_scl[index] = sample.SCL;
    band_snw[index] = sample.SNW;
    band_cld[index] = sample.CLD;
    band_clp[index] = sample.CLP;
    band_clm[index] = sample.CLM;
    band_vzm[index] = sample.viewZenithMean;
    band_vam[index] = sample.viewAzimuthMean;
    band_sza[index] = sample.sunZenithAngles;
    band_saa[index] = sample.sunAzimuthAngles;
    band_dm[index] = sample.dataMask;
  });

  return {
    b01: band_b01,
    b02: band_b02,
    b03: band_b03,
    b04: band_b04,
    b05: band_b05,
    b06: band_b06,
    b07: band_b07,
    b08: band_b08,
    b8a: band_b8a,
    b09: band_b09,
    b11: band_b11,
    b12: band_b12,
    aot: band_aot,
    scl: band_scl,
    snw: band_snw,
    cld: band_cld,
    clp: band_clp,
    clm: band_clm,
    vzm: band_vzm,
    vam: band_vam,
    sza: band_sza,
    saa: band_saa,
    dm: band_dm,
  };
}
2 Likes
3

Thanks! It works perfectly.

Do you know if there are any limits on time frames? It crashes on 1-year frames.

4

Hi @emidio ,

There’s a limit of the amount of data that can be processed. In this case half-year seems to be the limit of a single request.