diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 24e2c1204e6f697d6373e088cadb3165a994f5c5..bdb8804917beb7397f8bc78da33479b259a53e0c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,13 +23,29 @@ stages:
- Documentation
- Pip
-Test API:
+.static_analysis_base:
+ allow_failure: false
+
+Test Upload:
+ extends: .static_analysis_with_pip_install
+ stage: Test
+ except: [main]
+ script:
+ - python tests/test_upload.py
+ - DINAMIS_SDK_ACCESS_KEY=${CI_VAR_DINAMIS_SDK_ACCESS_KEY}
+ DINAMIS_SDK_SECRET_KEY=${CI_VAR_DINAMIS_SDK_SECRET_KEY}
+ python tests/test_upload.py
+
+Test Get:
+ extends: .static_analysis_with_pip_install
+ stage: Test
+ except: [main]
+ script:
+ - python tests/test_get.py
+
+Test Diff:
extends: .static_analysis_with_pip_install
stage: Test
- allow_failure: false
except: [main]
script:
- - python tests/all.py
- - DINAMIS_SDK_ACCESS_KEY=${CI_VAR_DINAMIS_SDK_ACCESS_KEY}
- DINAMIS_SDK_SECRET_KEY=${CI_VAR_DINAMIS_SDK_SECRET_KEY}
- python tests/all.py
+ - python tests/test_diff.py
diff --git a/README.md b/README.md
index 18b4aebdaca6778403e1e92b4739f60b01448fda..dd6e7a8fd6d233bb57e26f7391cae6379868ba57 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@
</p>
**Theia-dumper** enables to upload Spatio Temporal Assets Catalogs (STAC) on the
-THEIA-MTP geospatial data center.
+THEIA-MTD geospatial data center.
For more information read the [documentation](https://cdos-pub.pages.mia.inra.fr/theia-dumper).
diff --git a/pyproject.toml b/pyproject.toml
index 1082b9b1659cd7a4b3a1cc2e4c279c3aa0d7c426..05c6645e83b9da64c6e571b8803c30138e3e4e25 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,16 +4,18 @@ build-backend = "setuptools.build_meta"
[project]
name = "theia_dumper"
-version = "0.0.5"
description = "THEIA-MTP geospatial data publisher"
+dynamic = ["version"]
authors = [{ name = "Rémi Cresson", email = "remi.cresson@inrae.fr" }]
requires-python = ">=3.9"
dependencies = [
"setuptools",
"pystac",
"pystac_client",
- "dinamis_sdk>=0.4.0",
+ "dinamis_sdk>=0.4.1",
"requests",
+ "click",
+ "rich",
"rio-cogeo",
]
license = { text = "Apache-2.0" }
@@ -42,7 +44,7 @@ pretty = true
exclude = ["doc", "venv", ".venv"]
[tool.pylint]
-disable = "W1203,R0903,E0401,W0622,C0116,C0115"
+disable = "W1203,R0903,E0401,W0622,C0116,C0115,W0719"
[tool.pylint.MASTER]
ignore-paths = '^.venv'
diff --git a/tests/test_diff.py b/tests/test_diff.py
new file mode 100755
index 0000000000000000000000000000000000000000..3e0aebb0d04109fa25a1a2a7e6b3372c07643e6d
--- /dev/null
+++ b/tests/test_diff.py
@@ -0,0 +1,33 @@
+"""Test file."""
+
+import test_upload
+import pystac
+
+from theia_dumper import stac, diff
+
+
+col1, items = test_upload.create_items_and_collection(
+ relative=True, col_href="/tmp/collection.json"
+)
+col2 = col1.full_copy()
+
+item = items[0].full_copy()
+item.id += "_test"
+col2.add_item(item, item.id)
+
+item = items[0].full_copy()
+item.id += "_test_other"
+col1.add_item(item, item.id)
+
+diff.generate_items_diff(col1, col2)
+diff.collections_defs_are_different(col1, col2)
+
+COL1_FILEPATH = "/tmp/col1.json"
+col1.set_self_href(COL1_FILEPATH)
+col1.save(catalog_type=pystac.CatalogType.RELATIVE_PUBLISHED)
+
+diff.compare_local_and_upstream(
+ stac.StacTransactionsHandler(stac.DEFAULT_STAC_EP),
+ COL1_FILEPATH,
+ "costarica-sentinel-2-l3-seasonal-spectral-indices-M",
+)
diff --git a/tests/test_get.py b/tests/test_get.py
new file mode 100755
index 0000000000000000000000000000000000000000..652169c1b0ae095eaf0cb447aca561ff3a1aba74
--- /dev/null
+++ b/tests/test_get.py
@@ -0,0 +1,16 @@
+"""Test file."""
+
+from theia_dumper import stac, cli
+
+
+handler = stac.StacTransactionsHandler(
+ stac_endpoint=cli.DEFAULT_STAC_EP,
+)
+
+REMOTE_COL_ID = "spot-6-7-drs"
+handler.list_collections_display()
+handler.list_col_items_display(REMOTE_COL_ID)
+
+col_remote = handler.get_remote_col(REMOTE_COL_ID)
+
+col_items = handler.list_col_items(REMOTE_COL_ID)
diff --git a/tests/all.py b/tests/test_upload.py
similarity index 81%
rename from tests/all.py
rename to tests/test_upload.py
index 9143fb0e0a43aa0e1da2c66ed7d3fe97a91a928e..5355e784f7060e0249a5d969eca178e37f103d34 100755
--- a/tests/all.py
+++ b/tests/test_upload.py
@@ -12,32 +12,30 @@ import requests
from theia_dumper import stac
+DEFAULT_COL_HREF = "http://hello.fr/collections/collection-for-tests"
STAC_EP = "https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr"
-
-
-handler = stac.TransactionsHandler(
- stac_endpoint=STAC_EP,
- storage_endpoint="https://s3-data.meso.umontpellier.fr",
- storage_bucket="sm1-gdc-tests",
- assets_overwrite=True,
-)
-
IMAGE_HREF = (
"https://gitlab.orfeo-toolbox.org/orfeotoolbox/"
"otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
)
-
COL_ID = "collection-for-theia-dumper-tests"
items_ids = ["item_1", "item_2"]
-
RASTER_FILE1 = "/tmp/raster1.tif"
RASTER_FILE2 = "/tmp/raster2.tif"
+
+handler = stac.StacUploadTransactionsHandler(
+ stac_endpoint=STAC_EP,
+ storage_endpoint="https://s3-data.meso.umontpellier.fr",
+ storage_bucket="sm1-gdc-tests",
+ assets_overwrite=True,
+)
+
with open(RASTER_FILE1, "wb") as f:
r = requests.get(IMAGE_HREF, timeout=5)
f.write(r.content)
shutil.copyfile(RASTER_FILE1, RASTER_FILE2)
-COL_BBOX = [0, 0, 0, 0]
+COL_BBOX = [0.0, 0.0, 0.0, 0.0]
BBOX_ALL = [
3.6962018175925073,
43.547450099338604,
@@ -57,20 +55,26 @@ COORDS2 = [[coord + 5 for coord in coords] for coords in COORDS1]
def clear():
"""Clear all test items and collection."""
for item_id in items_ids:
- handler.delete(col_id=COL_ID, item_id=item_id)
- handler.delete(col_id=COL_ID)
+ handler.delete_item_or_col(col_id=COL_ID, item_id=item_id)
+ handler.delete_item_or_col(col_id=COL_ID)
-def check(expected_bbox):
- """Check collection extent."""
+def remote_col_test(expected_bbox):
+ """Run tests on a remote collection."""
api = pystac_client.Client.open(STAC_EP)
- extent = api.get_collection(COL_ID).extent.spatial.bboxes
- print(f"extent.spatial: {extent}")
+ col = api.get_collection(COL_ID)
+ extent = col.extent.spatial.bboxes
assert len(extent) == 1
assert tuple(extent[0]) == tuple(expected_bbox), (
f"expected BBOX: {expected_bbox}, got {extent[0]}"
)
+ # Check that assets are accessible once signed
+ for i in col.get_items():
+ assets = i.get_assets().values()
+ for asset in assets:
+ assert stac.asset_exists(asset.href)
+
def create_item(item_id: str):
"""Create a STAC item."""
@@ -98,15 +102,15 @@ def create_item(item_id: str):
return item
-def create_collection():
+def create_collection(col_href: str):
"""Create an empty STAC collection."""
spat_extent = pystac.SpatialExtent([COL_BBOX])
- temp_extent = pystac.TemporalExtent(intervals=[(None, None)])
+ temp_extent = pystac.TemporalExtent(intervals=[[None, None]]) # type: ignore
col = pystac.Collection(
id=COL_ID,
extent=pystac.Extent(spat_extent, temp_extent),
description="Some description",
- href="http://hello.fr/collections/collection-for-tests",
+ href=col_href,
providers=[
pystac.Provider("INRAE"),
],
@@ -114,13 +118,13 @@ def create_collection():
return col
-def create_items_and_collection(relative, items=None):
+def create_items_and_collection(relative, items=None, col_href=DEFAULT_COL_HREF):
"""Create two STAC items attached to one collection."""
# Create items
items = items or [create_item(item_id=item_id) for item_id in items_ids]
# Attach items to collection
- col = create_collection()
+ col = create_collection(col_href)
for item in items:
col.add_item(item)
if relative:
@@ -155,13 +159,13 @@ def test_item_collection():
print(f"Relative: {relative}")
# we need to create an empty collection before
- col = create_collection()
+ col = create_collection(DEFAULT_COL_HREF)
handler.publish_collection(collection=col)
with tempfile.NamedTemporaryFile() as tmp:
generate_item_collection(tmp.name, relative=relative)
handler.load_and_publish(tmp.name)
- check(BBOX_ALL)
+ remote_col_test(BBOX_ALL)
clear()
@@ -172,7 +176,7 @@ def test_collection():
with tempfile.TemporaryDirectory() as tmpdir:
generate_collection(tmpdir, relative=relative)
handler.load_and_publish(os.path.join(tmpdir, "collection.json"))
- check(BBOX_ALL)
+ remote_col_test(BBOX_ALL)
clear()
@@ -186,20 +190,17 @@ def test_collection_multipart():
tmpdir, relative=relative, items=[create_item(item_id)]
)
handler.load_and_publish(os.path.join(tmpdir, "collection.json"))
- check(BBOX_ALL)
+ remote_col_test(BBOX_ALL)
clear()
-def test_all():
- """Test all."""
- # test collection
+def _test_all():
test_collection()
- # test item collection
test_item_collection()
- # test collection (multi-part)
test_collection_multipart()
-test_all()
+if __name__ == "__main__":
+ _test_all()
diff --git a/theia_dumper/__init__.py b/theia_dumper/__init__.py
index 03970df13c1d94d863e594d033f6c21c77839c14..ec8539461cdb7667033272a0675ef109ae8da6c9 100644
--- a/theia_dumper/__init__.py
+++ b/theia_dumper/__init__.py
@@ -1 +1,3 @@
"""Theia dumper package."""
+
+__version__ = "0.1.0"
diff --git a/theia_dumper/cli.py b/theia_dumper/cli.py
index 8f6f70d2939ebdcc1cc9cfd5a00012e544a38ac6..1627dca554e23093acaf6dc3d4621969c817fcfa 100644
--- a/theia_dumper/cli.py
+++ b/theia_dumper/cli.py
@@ -1,10 +1,13 @@
"""Theia-dumper Command Line Interface."""
import click
-from .stac import TransactionsHandler, delete_stac_obj
-
-
-DEFAULT_STAC_EP = "https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr"
+from .stac import (
+ StacUploadTransactionsHandler,
+ StacTransactionsHandler,
+ DEFAULT_S3_EP,
+ DEFAULT_STAC_EP,
+)
+from . import diff
@click.group()
@@ -24,7 +27,7 @@ def theia_dumper() -> None:
"--storage_endpoint",
type=str,
help="Storage endpoint assets will be sent to",
- default="https://s3-data.meso.umontpellier.fr",
+ default=DEFAULT_S3_EP,
)
@click.option(
"-b",
@@ -48,13 +51,12 @@ def publish(
overwrite: bool,
):
"""Publish a STAC object (collection or item collection)."""
- handler = TransactionsHandler(
+ StacUploadTransactionsHandler(
stac_endpoint=stac_endpoint,
storage_endpoint=storage_endpoint,
storage_bucket=storage_bucket,
assets_overwrite=overwrite,
- )
- handler.load_and_publish(stac_obj_path)
+ ).load_and_publish(stac_obj_path)
@theia_dumper.command(context_settings={"show_default": True})
@@ -71,5 +73,72 @@ def delete(
col_id: str,
item_id: str,
):
- """Publish a STAC object (collection or item collection)."""
- delete_stac_obj(stac_endpoint=stac_endpoint, col_id=col_id, item_id=item_id)
+ """Delete a STAC object (collection or item)."""
+ StacTransactionsHandler(
+ stac_endpoint=stac_endpoint,
+ ).delete_item_or_col(col_id=col_id, item_id=item_id)
+
+
+@theia_dumper.command(context_settings={"show_default": True})
+@click.option(
+ "--stac_endpoint",
+ help="Endpoint to which STAC objects will be sent",
+ type=str,
+ default=DEFAULT_STAC_EP,
+)
+def list_cols(
+ stac_endpoint: str,
+):
+ """List collections."""
+ StacTransactionsHandler(
+ stac_endpoint=stac_endpoint,
+ ).list_collections_display()
+
+
+@theia_dumper.command(context_settings={"show_default": True})
+@click.option(
+ "--stac_endpoint",
+ help="Endpoint to which STAC objects will be sent",
+ type=str,
+ default=DEFAULT_STAC_EP,
+)
+@click.option("-c", "--col_id", type=str, help="STAC collection ID", required=True)
+@click.option(
+ "-m", "--max_items", type=int, help="Max number of items to display", default=20
+)
+def list_col_items(
+ stac_endpoint: str,
+ col_id: str,
+ max_items: int,
+):
+ """List collection items."""
+ StacTransactionsHandler(
+ stac_endpoint=stac_endpoint,
+ ).list_col_items_display(col_id=col_id, max_items=max_items)
+
+
+@theia_dumper.command(context_settings={"show_default": True})
+@click.option(
+ "--stac_endpoint",
+ help="Endpoint to which STAC objects will be sent",
+ type=str,
+ default=DEFAULT_STAC_EP,
+)
+@click.option("-p", "--col_path", type=str, help="Local collection path", required=True)
+@click.option(
+ "-r",
+ "--remote_id",
+ type=str,
+ help="Remote collection ID. If not specified, will use local collection ID",
+ required=False,
+)
+def collection_diff(
+ stac_endpoint: str,
+ col_path: str,
+ remote_id: str | None = None,
+):
+ """List collection items."""
+ handler = StacTransactionsHandler(
+ stac_endpoint=stac_endpoint,
+ )
+ diff.compare_local_and_upstream(handler, col_path, remote_id)
diff --git a/theia_dumper/diff.py b/theia_dumper/diff.py
new file mode 100644
index 0000000000000000000000000000000000000000..52032ee524b50657f9e17ecc48764c8a37dc2c6c
--- /dev/null
+++ b/theia_dumper/diff.py
@@ -0,0 +1,102 @@
+"""STAC diff tool."""
+
+from typing import Tuple, List, cast
+from pystac import Collection, Item
+from rich import print
+from .logger import logger
+
+from . import stac
+
+UNIQUE_SEP = "___"
+
+
+def collections_defs_are_different(col1: Collection, col2: Collection) -> bool:
+ """Compute the diff between 2 STAC collections."""
+
+ def fields_are_different(col1: Collection, col2: Collection, field_name: str):
+ recursive_fields = field_name.split(".")
+
+ f1 = col1
+ f2 = col2
+ for f in recursive_fields:
+ f1 = getattr(f1, f)
+ f2 = getattr(f2, f)
+
+ if f1 != f2:
+ logger.info(f"{field_name} is different: '{f1}' != '{f2}'")
+ return True
+ return False
+
+ fields = [
+ "extent.spatial.bboxes",
+ "extent.temporal.intervals",
+ "description",
+ "id",
+ "keywords",
+ "license",
+ "strategy",
+ "providers",
+ "title",
+ ]
+ return any(fields_are_different(col1, col2, field) for field in fields)
+
+
+def generate_items_diff(
+ col1: Collection, col2: Collection
+) -> Tuple[List[Item], List[Item]]:
+ """Compute the diff between 2 STAC collections.
+
+ Returns:
+ - list of items only in collection 1
+ - list of items only in collection 2
+ """
+
+ def item_get_unique(i: Item) -> str:
+ return i.id + UNIQUE_SEP + str(i.datetime.isoformat() if i.datetime else "")
+
+ col1_ids = [item_get_unique(i) for i in col1.get_items()]
+ col2_ids = [item_get_unique(i) for i in col2.get_items()]
+
+ only_in_1 = set(col1_ids) - set(col2_ids)
+ only_in_2 = set(col2_ids) - set(col1_ids)
+
+ def unique_retrieve_info(unique: str, col: Collection) -> Item:
+ id = unique.split(UNIQUE_SEP)[0]
+ item = col.get_item(id)
+ if not item:
+ raise Exception(f"Item {id} not found")
+ return item
+
+ list_only_in_1 = [unique_retrieve_info(unique, col1) for unique in only_in_1]
+ list_only_in_2 = [unique_retrieve_info(unique, col2) for unique in only_in_2]
+
+ return list_only_in_1, list_only_in_2
+
+
+def compare_local_and_upstream(
+ handler: stac.StacTransactionsHandler,
+ local_col_path: str,
+ remote_col_id: str | None = None,
+):
+ """Compare a local and a remote collection.
+
+ Args:
+ handler (stac.StacTransactionHandler): object to handle the connection
+ local_col_path (str): path to local collection path
+ remote_col_id (str | None, optional): Remote collection identifier.
+ If unset, will take the same id as the local collection
+ """
+ col_local = cast(Collection, stac.load_stac_obj(obj_pth=local_col_path))
+ if not remote_col_id:
+ remote_col_id = col_local.id
+ col_remote = handler.get_remote_col(remote_col_id)
+
+ only_local, only_remote = generate_items_diff(col_local, col_remote)
+
+ collections_defs_are_different(col_local, col_remote)
+
+ print(f"Only local ({len(only_local)}):")
+ print(only_local[:20])
+
+ print(f"Only remote ({len(only_remote)}):")
+ print(only_remote[:20])
diff --git a/theia_dumper/stac.py b/theia_dumper/stac.py
index 1a9026849cf4ab2350b36485a84f68e22b63b9ab..6ea96ee05e08c198c3cecdbebb048c0781dede24 100644
--- a/theia_dumper/stac.py
+++ b/theia_dumper/stac.py
@@ -1,9 +1,11 @@
"""STAC stuff."""
import os
+import re
from dataclasses import dataclass
-from typing import List
+from typing import List, cast
from urllib.parse import urljoin
+import operator
import dinamis_sdk
import pystac
@@ -14,6 +16,9 @@ from requests.adapters import HTTPAdapter, Retry
from .logger import logger
+DEFAULT_STAC_EP = "https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr"
+DEFAULT_S3_EP = "https://s3-data.meso.umontpellier.fr"
+
class STACObjectUnresolved(Exception):
"""Unresolved STAC object exception."""
@@ -23,6 +28,14 @@ class UnconsistentCollectionIDs(Exception):
"""Inconsistent STAC collection exception."""
+def _check_naming_is_compliant(s: str, allow_dot=False):
+ _s = re.sub(r"[-|_]", r"", s)
+ if allow_dot:
+ _s = re.sub(r"\.", r"", _s)
+ if not _s.isalnum():
+ raise Exception(f"{_s} does not only contain alphanumeric or - or _ chars")
+
+
def create_session():
"""Create a requests session."""
sess = requests.Session()
@@ -54,6 +67,16 @@ def create_session():
return sess
+def asset_exists(url: str) -> bool:
+ """Check that the item provided in parameter exists and is accessible."""
+ sess = create_session()
+ res = sess.get(dinamis_sdk.sign(url), stream=True)
+ if res.status_code == 200:
+ logger.info("Asset %s already exists. Skipping.", url)
+ return True
+ return False
+
+
def post_or_put(url: str, data: dict):
"""Post or put data to url."""
headers = dinamis_sdk.get_headers()
@@ -78,7 +101,7 @@ def post_or_put(url: str, data: dict):
raise e
-def load(obj_pth):
+def load_stac_obj(obj_pth: str) -> Collection | ItemCollection | Item:
"""Load a STAC object serialized on disk."""
for obj_name, cls in {
"collection": Collection,
@@ -108,7 +131,7 @@ def get_assets_root_dir(items: List[Item]) -> str:
def check_items_collection_id(items: List[Item]):
- """Check that items collection_id is unique."""
+ """Check that items have the same collection_id."""
if len(set(item.collection_id for item in items)) > 1:
raise UnconsistentCollectionIDs("Collection ID must be the same for all items!")
@@ -125,37 +148,92 @@ def get_col_items(col: Collection) -> List[Item]:
"""Retrieve collection items."""
col_href = get_col_href(col=col)
return [
- load(
- os.path.join(os.path.dirname(col_href), link.href[2:])
- if link.href.startswith("./")
- else link.href
+ cast(
+ Item,
+ load_stac_obj(
+ os.path.join(os.path.dirname(col_href), link.href[2:])
+ if link.href.startswith("./")
+ else link.href
+ ),
)
for link in col.links
if link.rel == "item"
]
-def delete_stac_obj(stac_endpoint: str, col_id: str, item_id: str | None = None):
- """Delete an item or a collection."""
- logger.info("Deleting %s%s", col_id, f"/{item_id}" if item_id else "")
- if item_id:
- url = f"{stac_endpoint}/collections/{col_id}/items/{item_id}"
- else:
- url = f"{stac_endpoint}/collections/{col_id}"
- resp = requests.delete(
- url,
- headers=dinamis_sdk.get_headers(),
- timeout=5,
- )
- if resp.status_code != 200:
- logger.warning("Deletion failed (%s)", resp.text)
+@dataclass
+class StacTransactionsHandler:
+ """Handle STAC and storage transactions."""
+
+ stac_endpoint: str
+
+ def delete_item_or_col(self, col_id: str, item_id: str | None = None):
+ """Delete an item or a collection."""
+ logger.info("Deleting %s%s", col_id, f"/{item_id}" if item_id else "")
+ if item_id:
+ url = f"{self.stac_endpoint}/collections/{col_id}/items/{item_id}"
+ else:
+ url = f"{self.stac_endpoint}/collections/{col_id}"
+ resp = requests.delete(
+ url,
+ headers=dinamis_sdk.get_headers(),
+ timeout=5,
+ )
+ if resp.status_code != 200:
+ logger.warning("Deletion failed (%s)", resp.text)
+
+ def list_collections(self):
+ """List collections."""
+ logger.info("Listing collections")
+ url = f"{self.stac_endpoint}/collections"
+ resp = requests.get(
+ url,
+ timeout=5,
+ )
+ if resp.status_code != 200:
+ logger.warning("Get failed (%s)", resp.text)
+ cols = resp.json()["collections"]
+ cols.sort(key=operator.itemgetter("id"))
+ return cols
+
+ def list_collections_display(self):
+ """Display in terminal a list of available collections."""
+ cols = self.list_collections()
+ print(f"{len(cols)} collections available")
+ for col in cols:
+ print("\t" + col["id"])
+
+ def list_col_items(self, col_id: str, max_items=10):
+ """List items in a collection."""
+ logger.info("Listing %s items", col_id)
+ api = pystac_client.Client.open(
+ self.stac_endpoint,
+ modifier=dinamis_sdk.sign_inplace,
+ )
+ res = api.search(collections=[col_id], max_items=max_items)
+ items = list(res.items())
+ return items
+
+ def list_col_items_display(self, col_id: str, max_items=10):
+ """Display in terminal items in a collection."""
+ items = self.list_col_items(col_id, max_items=max_items)
+ print(f"{len(items)} items found:")
+ for item in items:
+ print("\t" + item.id)
+
+ def get_remote_col(self, col_id) -> Collection:
+ """Retrieve a remote collection."""
+ api = pystac_client.Client.open(
+ self.stac_endpoint,
+ modifier=dinamis_sdk.sign_inplace,
+ )
+ return api.get_collection(col_id)
@dataclass
-class TransactionsHandler:
+class StacUploadTransactionsHandler(StacTransactionsHandler):
"""Handle STAC and storage transactions."""
- stac_endpoint: str
storage_endpoint: str
storage_bucket: str
assets_overwrite: bool
@@ -165,23 +243,27 @@ class TransactionsHandler:
col_id = item.collection_id
target_root_dir = urljoin(self.storage_endpoint, self.storage_bucket)
+ _check_naming_is_compliant(self.storage_bucket)
+ _check_naming_is_compliant(item.id)
+
# Upload assets files
for _, asset in item.assets.items():
local_filename = asset.href
logger.debug("Local file: %s", local_filename)
target_url = local_filename.replace(assets_root_dir, target_root_dir)
+
+ _check_naming_is_compliant(
+ target_url.replace(target_root_dir + "/", ""), allow_dot=True
+ )
logger.debug("Target file: %s", target_url)
# Skip when target file exists and overwrite is not enabled
if not self.assets_overwrite:
- sess = create_session()
- res = sess.get(dinamis_sdk.sign(target_url), stream=True)
- if res.status_code == 200:
- logger.info("Asset %s already exists. Skipping.", target_url)
+ if asset_exists(target_url):
continue
# Upload file
- logger.info("Uploading %s ...", local_filename)
+ logger.info("Uploading %s to %s...", local_filename, target_url)
try:
dinamis_sdk.push(local_filename=local_filename, target_url=target_url)
except Exception as e:
@@ -229,6 +311,7 @@ class TransactionsHandler:
def publish_collection(self, collection: Collection):
"""Publish an empty collection."""
+ _check_naming_is_compliant(collection.id)
post_or_put(
url=urljoin(self.stac_endpoint, "/collections"), data=collection.to_dict()
)
@@ -246,7 +329,7 @@ class TransactionsHandler:
def load_and_publish(self, obj_pth: str):
"""Load and publish the serialized STAC object."""
- obj = load(obj_pth=obj_pth)
+ obj = load_stac_obj(obj_pth=obj_pth)
if isinstance(obj, Collection):
self.publish_collection_with_items(collection=obj)
elif isinstance(obj, ItemCollection):
@@ -255,9 +338,3 @@ class TransactionsHandler:
raise TypeError(
f"Invalid type, must be ItemCollection or Collection (got {type(obj)})"
)
-
- def delete(self, col_id: str, item_id: str | None = None):
- """Delete an item or a collection."""
- delete_stac_obj(
- stac_endpoint=self.stac_endpoint, col_id=col_id, item_id=item_id
- )