Release v0.1.2

Fixes #10 once and for all.

- Replace `common_prefix` by `common_path` that was causing problems (for instance if `/tmp/folder` and `/tmp/folder1`)
- Use harmonized urls in object full url creation
- Fix HTTP error display on Item push failure
- Add pytest and code coverage

See merge request !18

.gitignore

@@ -5,6 +5,8 @@ build
 dist
 draft
 .vscode
+.coverage
 public*/
 *venv/
+*.xml
 

.gitlab-ci.yml

@@ -10,6 +10,7 @@ variables:
   IMPL_MODULE_NAME: theia_dumper
 
   PYTHON_IMG: python:3.12-slim
+  PACKAGE_INSTALL_EXTRAS: "[test]"
 
   DOC_BUILD_SELF: true
 
@@ -36,16 +37,24 @@ Test Upload:
       DINAMIS_SDK_SECRET_KEY=${CI_VAR_DINAMIS_SDK_SECRET_KEY}
       python tests/test_upload.py
 
-Test Get:
+Pytest:
   extends: .static_analysis_with_pip_install
   stage: Test
   except: [main]
   script:
-    - python tests/test_get.py
-
-Test Diff:
-  extends: .static_analysis_with_pip_install
-  stage: Test
-  except: [main]
-  script:
-    - python tests/test_diff.py
+    - DINAMIS_SDK_ACCESS_KEY=${CI_VAR_DINAMIS_SDK_ACCESS_KEY}
+      DINAMIS_SDK_SECRET_KEY=${CI_VAR_DINAMIS_SDK_SECRET_KEY}
+      coverage run -m pytest -rsv --junit-xml=report.xml
+    - coverage report
+    - coverage xml
+    - coverage html
+  coverage: '/^TOTAL.+?(\d+\%)$/'
+  artifacts:
+    paths:
+      - htmlcov/
+    when: always
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+      junit: report.xml

pyproject.toml

@@ -28,6 +28,8 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
     "Operating System :: OS Independent",
 ]
+[project.optional-dependencies]
+test = ["pytest", "coverage"]
 
 [project.scripts]
 theia_dumper = "theia_dumper.cli:theia_dumper"
@@ -44,7 +46,7 @@ pretty = true
 exclude = ["doc", "venv", ".venv"]
 
 [tool.pylint]
-disable = "W1203,R0903,E0401,W0622,C0116,C0115,W0719"
+disable = "W1203,R0903,E0401,W0622,C0116,C0115,W0718,W0719"
 
 [tool.pylint.MASTER]
 ignore-paths = '^.venv'

tests/test_diff.py

 """Test file."""
 
-import test_upload
 import pystac
+import test_upload
+import utils
+
+from theia_dumper import diff, stac
+
+
+def test_diff():
+    """Test diff."""
+
+    utils.set_secret_key_env()
 
-from theia_dumper import stac, diff
+    col1, items = test_upload.create_items_and_collection(
+        relative=True, col_href="/tmp/collection.json"
+    )
+    col2 = col1.full_copy()
 
+    item = items[0].full_copy()
+    item.id += "_test"
+    col2.add_item(item, item.id)
 
-col1, items = test_upload.create_items_and_collection(
-    relative=True, col_href="/tmp/collection.json"
-)
-col2 = col1.full_copy()
+    item = items[0].full_copy()
+    item.id += "_test_other"
+    col1.add_item(item, item.id)
 
-item = items[0].full_copy()
-item.id += "_test"
-col2.add_item(item, item.id)
+    diff.generate_items_diff(col1, col2)
+    diff.collections_defs_are_different(col1, col2)
 
-item = items[0].full_copy()
-item.id += "_test_other"
-col1.add_item(item, item.id)
+    col1_filepath = "/tmp/col1.json"
+    col1.set_self_href(col1_filepath)
+    col1.save(catalog_type=pystac.CatalogType.RELATIVE_PUBLISHED)
 
-diff.generate_items_diff(col1, col2)
-diff.collections_defs_are_different(col1, col2)
+    diff.compare_local_and_upstream(
+        stac.StacTransactionsHandler(stac.DEFAULT_STAC_EP),
+        col1_filepath,
+        "costarica-sentinel-2-l3-seasonal-spectral-indices-M",
+    )
 
-COL1_FILEPATH = "/tmp/col1.json"
-col1.set_self_href(COL1_FILEPATH)
-col1.save(catalog_type=pystac.CatalogType.RELATIVE_PUBLISHED)
 
-diff.compare_local_and_upstream(
-    stac.StacTransactionsHandler(stac.DEFAULT_STAC_EP),
-    COL1_FILEPATH,
-    "costarica-sentinel-2-l3-seasonal-spectral-indices-M",
-)
+if __name__ == "__main__":
+    test_diff()

tests/test_get.py

 """Test file."""
 
-from theia_dumper import stac, cli
+import utils
 
+from theia_dumper import cli, stac
 
-handler = stac.StacTransactionsHandler(
-    stac_endpoint=cli.DEFAULT_STAC_EP,
-)
 
-REMOTE_COL_ID = "spot-6-7-drs"
-handler.list_collections_display()
-handler.list_col_items_display(REMOTE_COL_ID)
+def test_get():
+    """Test get."""
+    utils.set_secret_key_env()
+    handler = stac.StacTransactionsHandler(
+        stac_endpoint=cli.DEFAULT_STAC_EP,
+    )
 
-col_remote = handler.get_remote_col(REMOTE_COL_ID)
+    remote_col_id = "spot-6-7-drs"
+    handler.list_collections_display()
+    handler.list_col_items_display(remote_col_id)
 
-col_items = handler.list_col_items(REMOTE_COL_ID)
+    handler.get_remote_col(remote_col_id)
+
+    handler.list_col_items(remote_col_id)
+
+
+if __name__ == "__main__":
+    test_get()

tests/test_upload.py

@@ -8,9 +8,12 @@ from datetime import datetime
 import pystac
 import pystac_client
 import requests
+import utils
+import pytest
 
 from theia_dumper import stac
 
+utils.set_secret_key_env()
 
 DEFAULT_COL_HREF = "http://hello.fr/collections/collection-for-tests"
 STAC_EP = "https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr"
@@ -21,7 +24,8 @@ IMAGE_HREF = (
 COL_ID = "collection-for-theia-dumper-tests"
 items_ids = ["item_1", "item_2"]
 RASTER_FILE1 = "/tmp/raster1.tif"
-RASTER_FILE2 = "/tmp/folder/raster2.tif"
+RASTER_FILE2 = "/tmp/folder1/raster2.tif"
+RASTER_FILE3 = "/tmp/folder/raster3.tif"
 
 handler = stac.StacUploadTransactionsHandler(
     stac_endpoint=STAC_EP,
@@ -35,6 +39,8 @@ with open(RASTER_FILE1, "wb") as f:
     f.write(r.content)
 os.makedirs(os.path.dirname(RASTER_FILE2), exist_ok=True)
 shutil.copyfile(RASTER_FILE1, RASTER_FILE2)
+os.makedirs(os.path.dirname(RASTER_FILE3), exist_ok=True)
+shutil.copyfile(RASTER_FILE1, RASTER_FILE3)
 
 COL_BBOX = [0.0, 0.0, 0.0, 0.0]
 BBOX_ALL = [
@@ -97,6 +103,7 @@ def create_item(item_id: str):
         assets={
             "ndvi": pystac.Asset(href=RASTER_FILE1),
             "crswir": pystac.Asset(href=RASTER_FILE2),
+            "ndwi": pystac.Asset(href=RASTER_FILE3),
         },
     )
 
@@ -155,53 +162,57 @@ def generate_item_collection(file_pth, relative=True):
     icol.save_object(file_pth)
 
 
-def test_item_collection():
+@pytest.mark.parametrize("assets_overwrite", [False, True])
+@pytest.mark.parametrize("relative", [False, True])
+def test_item_collection(assets_overwrite, relative):
     """Test item collection."""
-    for relative in [True, False]:
-        print(f"Relative: {relative}")
 
-        # we need to create an empty collection before
-        col = create_collection(DEFAULT_COL_HREF)
-        handler.publish_collection(collection=col)
+    handler.assets_overwrite = assets_overwrite
+    # we need to create an empty collection before
+    col = create_collection(DEFAULT_COL_HREF)
+    handler.publish_collection(collection=col)
 
-        with tempfile.NamedTemporaryFile() as tmp:
-            generate_item_collection(tmp.name, relative=relative)
-            handler.load_and_publish(tmp.name)
-            remote_col_test(BBOX_ALL)
-            clear()
+    with tempfile.NamedTemporaryFile() as tmp:
+        generate_item_collection(tmp.name, relative=relative)
+        handler.load_and_publish(tmp.name)
+        remote_col_test(BBOX_ALL)
+        clear()
 
 
-def test_collection():
+@pytest.mark.parametrize("assets_overwrite", [False, True])
+@pytest.mark.parametrize("relative", [False, True])
+def test_collection(assets_overwrite, relative):
     """Test collection."""
-    for relative in [True, False]:
-        print(f"\nRelative: {relative}")
-        with tempfile.TemporaryDirectory() as tmpdir:
-            generate_collection(tmpdir, relative=relative)
-            handler.load_and_publish(os.path.join(tmpdir, "collection.json"))
-            remote_col_test(BBOX_ALL)
-            clear()
+    handler.assets_overwrite = assets_overwrite
+    with tempfile.TemporaryDirectory() as tmpdir:
+        generate_collection(tmpdir, relative=relative)
+        handler.load_and_publish(os.path.join(tmpdir, "collection.json"))
+        remote_col_test(BBOX_ALL)
+        clear()
 
 
-def test_collection_multipart():
+@pytest.mark.parametrize("assets_overwrite", [False, True])
+@pytest.mark.parametrize("relative", [False, True])
+def test_collection_multipart(assets_overwrite, relative):
     """Test collection."""
-    for relative in [True, False]:
-        print(f"\nRelative: {relative}")
-        for item_id in items_ids:
-            with tempfile.TemporaryDirectory() as tmpdir:
-                generate_collection(
-                    tmpdir, relative=relative, items=[create_item(item_id)]
-                )
-                handler.load_and_publish(os.path.join(tmpdir, "collection.json"))
-        remote_col_test(BBOX_ALL)
-        clear()
+    print(f"\nRelative: {relative}")
+    handler.assets_overwrite = assets_overwrite
+    for item_id in items_ids:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            generate_collection(tmpdir, relative=relative, items=[create_item(item_id)])
+            handler.load_and_publish(os.path.join(tmpdir, "collection.json"))
+    remote_col_test(BBOX_ALL)
+    clear()
 
 
 def _test_all():
-    test_collection()
-
-    test_item_collection()
+    for relative in [False, True]:
+        for assets_overwrite in [False, True]:
+            handler.assets_overwrite = assets_overwrite
 
-    test_collection_multipart()
+            test_collection(assets_overwrite, relative)
+            test_item_collection(assets_overwrite, relative)
+            test_collection_multipart(assets_overwrite, relative)
 
 
 if __name__ == "__main__":

tests/utils.py

+"""Utils file."""
+
+import os
+
+import dinamis_sdk
+import dinamis_sdk.settings
+
+
+def set_secret_key_env():
+    """Test diff."""
+    if "DINAMIS_SDK_ACCESS_KEY" in os.environ:
+        dinamis_sdk.settings.ENV.dinamis_sdk_access_key = os.environ.get(
+            "DINAMIS_SDK_ACCESS_KEY"
+        )
+    if "DINAMIS_SDK_SECRET_KEY" in os.environ:
+        dinamis_sdk.settings.ENV.dinamis_sdk_secret_key = os.environ.get(
+            "DINAMIS_SDK_SECRET_KEY"
+        )

theia_dumper/__init__.py

 """Theia dumper package."""
 
-__version__ = "0.1.1"
+__version__ = "0.1.2"

theia_dumper/stac.py

@@ -13,6 +13,7 @@ import pystac_client
 import requests
 from pystac import Collection, Item, ItemCollection
 from requests.adapters import HTTPAdapter, Retry
+from rich.pretty import pretty_repr
 
 from .logger import logger
 
@@ -42,26 +43,19 @@ def create_session():
     """Create a requests session."""
     sess = requests.Session()
     retries = Retry(
-        total=5,
+        total=3,
         backoff_factor=1,
         status_forcelist=[
-            400,
-            403,
             408,
-            410,
             419,
-            421,
-            422,
-            424,
             425,
-            429,
             500,
             502,
             503,
             504,
-            505,
         ],
         allowed_methods=frozenset(["PUT", "POST"]),
+        raise_on_status=False,
     )
     adapter = HTTPAdapter(max_retries=retries)
     sess.mount("http://", adapter=adapter)
@@ -69,23 +63,30 @@ def create_session():
     return sess
 
 
-def asset_exists(url: str) -> bool:
-    """Check that the item provided in parameter exists and is accessible."""
+def asset_exists(asset_url: str) -> None | str:
+    """Check that the item provided in parameter exists and is accessible.
+
+    If asset exists, returns asset url
+    """
     sess = create_session()
-    res = sess.get(dinamis_sdk.sign(url), stream=True)
+    asset_url_signed = dinamis_sdk.sign(asset_url)
+    res = sess.get(asset_url_signed, stream=True)
     if res.status_code == 200:
-        logger.info("Asset %s already exists. Skipping.", url)
-        return True
-    return False
+        logger.info("Asset %s already exists.", asset_url)
+        return asset_url
+    return None
 
 
 def post_or_put(url: str, data: dict):
     """Post or put data to url."""
     headers = dinamis_sdk.get_headers()
     sess = create_session()
+
     resp = sess.post(url, json=data, headers=headers, timeout=10)
+
     if resp.status_code == 409:
         # Exists, so update
+        logger.info(f"Item at {url} already exists, doing a PUT")
         resp = sess.put(
             f"{url}/{data['id']}",
             json=data,
@@ -99,7 +100,10 @@ def post_or_put(url: str, data: dict):
     try:
         resp.raise_for_status()
     except Exception as e:
-        logger.error("Server returned: %s", resp.text)
+        try:
+            logger.error("Server returned: %s", pretty_repr(resp.json()))
+        except Exception:
+            logger.error("Server returned: %s", resp.text)
         raise e
 
 
@@ -127,12 +131,12 @@ def get_assets_root_dir(items: List[Item]) -> str:
 
     If the the common prefix is not a folder (/tmp/test1/a.tif, /tmp/test2/b.tif), returns /tmp.
     """
-    prefix = os.path.commonprefix(
+    prefix = os.path.commonpath(
         [asset.href for item in items for asset in item.assets.values()]
     )
     if os.path.isdir(prefix):
-        return prefix
-    return os.path.dirname(prefix)
+        return prefix + "/"
+    return os.path.dirname(prefix) + "/"
 
 
 def check_items_collection_id(items: List[Item]):
@@ -259,11 +263,13 @@ class StacUploadTransactionsHandler(StacTransactionsHandler):
             local_filename = asset.href
             logger.debug("Local file: %s", local_filename)
 
-            target_url = local_filename.replace(assets_root_dir, tgt_bucket_root_url)
+            file_relative_path = local_filename.replace(assets_root_dir, "")
+            target_url = urljoin(tgt_bucket_root_url, file_relative_path)
+            print(target_url)
 
             # Check that url part after storage bucket is compliant
             _check_naming_is_compliant(
-                target_url.replace(tgt_bucket_root_url, ""),
+                file_relative_path,
                 allow_dot=True,
                 allow_slash=True,
             )
@@ -271,7 +277,8 @@ class StacUploadTransactionsHandler(StacTransactionsHandler):
 
             # Skip when target file exists and overwrite is not enabled
             if not self.assets_overwrite:
-                if asset_exists(target_url):
+                if asset_href := asset_exists(target_url):
+                    asset.href = asset_href
                     continue
 
             # Upload file