I posted a question on the Apps / Services forum (Internal Server Error when downloading HLS image subsets) about an issue I had accessing Harmonized Landsat Sentinel (HLS) imagery and never got a responce to help me solve the problem so I trying on the EO Products / Scripts category.
I am trying to download subsets of Harmonized Landsat Sentinel (HLS) images, and it appears that one of the images, day 180 in 2023 (June 29, 2023), is causing an “Internal Server Error.” A couple of months ago, when I ran a similar script, that image was downloaded just fine. Can you please let me know if there is anything I can do to resolve this issue or if you need additional information?
Here is the error:
500 Server Error: Internal Server Error for url: https://services-uswest2.sentinel-hub.com/api/v1/process
Server response: “{“status”: 500, “reason”: “Internal Server Error”, “message”: “Illegal request to s3://lp-prod-protected/HLSL30.020/HLS.L30.T18TXQ.2023180T153815.v2.0/HLS.L30.T18TXQ.2023180T153815.v2.0.B01.tif. HTTP Status: ‘404’”, “code”: “RENDERER_EXCEPTION”}”
This is the code I am using:
get_ipython().run_line_magic('matplotlib', 'inline')
import datetime
import numpy as np
from rasterio import features, mask, transform, rasterio
import matplotlib.pyplot as plt
from aenum import MultiValueEnum
from matplotlib.colors import BoundaryNorm, ListedColormap
import os
import subprocess
import geopandas as gpd
import math
from sentinelhub import CRS, BBox, DataCollection, SHConfig
from eolearn.core import EOWorkflow, FeatureType, LoadTask, OutputTask, SaveTask, linearly_connect_tasks, MergeFeatureTask
from eolearn.io import SentinelHubDemTask, SentinelHubEvalscriptTask, SentinelHubInputTask, ExportToTiffTask
#shapefile = 'PaddocksUTM18N_21August2024_Aggregated.shp'
shapefile = 'PaddocksUTM18N_21August2024_Aggregated.shp'
eopatch_path = "Paddocks2023FarmAggregate21August2024/"
epsg = "EPSG:32618"
time_interval = ("2023-04-25", "2023-11-15") # time interval of downloaded data
resolution = 30 # resolution of the request (in metres)
vt_bbox = [624360, 4732860 , 777150, 4991340] # Coordinates for VT [xmin, ymin, xmax, ymax]
expansion_dist = 100
time_difference = datetime.timedelta(hours=12)
def adjust_bounding_box(xmin, ymin, xmax, ymax):
# Adjust the bounding box coordinates to ensure they are all evenly divisible by thirty.
# Adjust xmin and ymin to be the largest number less than or equal to them that is divisible by 30
xmin_adjusted = (xmin // 30) * 30
ymin_adjusted = (ymin // 30) * 30
# Adjust xmax and ymax to be the smallest number greater than or equal to them that is divisible by 30
# by adding 29 (to ensure the number is rounded up to the next divisible by 30 if not already divisible) and then doing integer division
xmax_adjusted = ((xmax + 29) // 30) * 30
ymax_adjusted = ((ymax + 29) // 30) * 30
return [xmin_adjusted, ymin_adjusted, xmax_adjusted, ymax_adjusted]
# In case you put the credentials into the configuration file you can leave this unchanged
CLIENT_ID = ""
CLIENT_SECRET = ""
config = SHConfig()
if CLIENT_ID and CLIENT_SECRET:
config.sh_client_id = CLIENT_ID
config.sh_client_secret = CLIENT_SECRET
if config.sh_client_id == "" or config.sh_client_secret == "" or config.instance_id == "":
print("Warning! To use Sentinel Hub services, please provide the credentials (client ID and client secret).")
gdf = gpd.read_file(shapefile)
bands_evalscript = """
//VERSION=3
function setup() {
return {
input: [{
bands: ["CoastalAerosol", "Blue", "Green", "Red", "NIR_Narrow", "SWIR1", "SWIR2", "Cirrus",\
"QA", "dataMask"],
units: "DN"
}],
output: [{
id: "ms_data",
bands: 8,
sampleType: SampleType.UINT16
}, {
id: "quality_bands",
bands: 2,
sampleType: SampleType.INT16
}]
}
}
function evaluatePixel(sample) {
bandsDN = [sample.CoastalAerosol, sample.Blue, sample.Green, sample.Red, sample.NIR_Narrow,
sample.SWIR1, sample.SWIR2, sample.Cirrus];
qualityBands = [sample.QA, sample.dataMask]
return {
ms_data: bandsDN,
quality_bands: qualityBands
}
}
"""
# this will add all Sentinel multispectral bands as DN (int) not reflectance (float)
add_ms_bands = SentinelHubEvalscriptTask(
features=[(FeatureType.DATA, "ms_data")],
evalscript=bands_evalscript,
data_collection=DataCollection.HARMONIZED_LANDSAT_SENTINEL,
resolution=resolution,
time_difference=time_difference,
config=config,
max_threads=3
)
# this will add all Sentinel pixel quality bands as DN (int) not reflectance (float)
add_quality_bands = SentinelHubEvalscriptTask(
features=[(FeatureType.DATA, "quality_bands")],
evalscript=bands_evalscript,
data_collection=DataCollection.HARMONIZED_LANDSAT_SENTINEL,
resolution=resolution,
time_difference=time_difference,
config=config,
max_threads=3
)
farm_list = []
for i in range(len(gdf)):
print(f"\rProcessing aggregate {i+1} of {len(gdf)}", end='', flush=True)
# Get the current polygon
polygon = gdf.loc[i, "geometry"]
# Extract the polygon's coordinates
x_coords, y_coords = polygon.exterior.coords.xy
# Get the filename by concatenating the "farm" and "name" attributes
farm = gdf.loc[i, 'Farm']
farm_list.append(farm)
xmin = int(min(x_coords) - expansion_dist)
xmax = int(max(x_coords) + expansion_dist)
ymin = int(min(y_coords) - expansion_dist)
ymax = int(max(y_coords) + expansion_dist)
# region of interest
roi_coordinates = adjust_bounding_box(xmin, ymin, xmax, ymax)
roi_bbox = BBox(roi_coordinates, crs=CRS(epsg))
save = SaveTask(eopatch_path + "numpyHLS", overwrite_permission=2, compress_level=0)
output_task = OutputTask(farm)
#workflow_nodes = linearly_connect_tasks(add_indices, add_ms_bands, save, output_task)
workflow_nodes = linearly_connect_tasks(add_ms_bands, add_quality_bands, save, output_task)
workflow = EOWorkflow(workflow_nodes)
result = workflow.execute(
{
workflow_nodes[0]: {"bbox": roi_bbox, "time_interval": time_interval},
workflow_nodes[-2]: {"eopatch_folder": farm},
}
)```