Fix bugs and clarify logging

- Improve items publish and fix asset href
- Fix logging logleve from env
- Improve dinamis post error retrieval
- Redirect logger stream to stdout

See merge request !5

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "theia_dumper"
-version = "0.0.1"
+version = "0.0.2"
 description = "THEIA-MTP geospatial data publisher"
 authors = [
     { name = "Rémi Cresson", email = "remi.cresson@inrae.fr" },
@@ -13,7 +13,7 @@ requires-python = ">=3.9"
 dependencies = [
     "pystac",
     "pystac_client",
-    "dinamis_sdk > 0.3.0",
+    "dinamis_sdk==0.3.3",
     "requests",
     "rio-cogeo"
 ]

tests/all.py

@@ -10,7 +10,7 @@ from theia_dumper import stac
 handler = stac.TransactionsHandler(
     stac_endpoint="https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr",
     storage_endpoint="https://s3-data.meso.umontpellier.fr",
-    storage_bucket="sm1-gdc",
+    storage_bucket="sm1-gdc-tests",
     assets_overwrite=True
 )
 

theia_dumper/logger.py

 """Logging stuff."""
 import logging
-logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
+import sys
+import os
+logging.basicConfig(format='%(levelname)s:%(message)s', level=os.environ.get("LOGLEVEL", "INFO"), stream=sys.stdout)
 logger = logging.getLogger(__name__)

theia_dumper/stac.py

@@ -55,7 +55,11 @@ def post_or_put(url: str, data: dict):
         if not resp.status_code == 404:
             resp.raise_for_status()
     else:
-        resp.raise_for_status()
+        try:
+            resp.raise_for_status()
+        except Exception as e:
+            logger.error(eval(resp.content)["detail"])
+            raise(e)
 
 
 def load(obj_pth):
@@ -135,10 +139,10 @@ class TransactionsHandler:
             self.storage_bucket,
             col_id
         )
-        local_assets_files = [asset.href for asset in item.assets.values()]
 
         # Upload assets files
-        for local_filename in local_assets_files:
+        for asset_name, asset in item.assets.items():
+            local_filename = asset.href
             logger.debug("Local file: %s", local_filename)
             target_url = local_filename.replace(
                 assets_root_dir,
@@ -161,29 +165,32 @@ class TransactionsHandler:
 
             # Upload file
             logger.info("Uploading %s ...", local_filename)
-            dinamis_sdk.push(
-                local_filename=local_filename,
-                target_url=target_url
-            )
+            try:
+                dinamis_sdk.push(
+                    local_filename=local_filename,
+                    target_url=target_url
+                )
+            except Exception as e:
+                logger.error(e)
+                raise(e)
 
             # Update assets hrefs
             logger.debug("Updating assets HREFs ...")
-            for asset_name in item.assets:
-                item.assets[asset_name].href = target_url
-
-            # Push item
-            logger.info(
-                "Publishing item \"%s\" in collection \"%s\"",
-                item.id,
-                col_id
-            )
-            post_or_put(
-                urljoin(
-                    self.stac_endpoint,
-                    f"collections/{col_id}/items"
-                ),
-                item.to_dict(transform_hrefs=False)
-            )
+            asset.href = target_url
+
+        # Push item
+        logger.info(
+            "Publishing item \"%s\" in collection \"%s\"",
+            item.id,
+            col_id
+        )
+        post_or_put(
+            urljoin(
+                self.stac_endpoint,
+                f"collections/{col_id}/items"
+            ),
+            item.to_dict(transform_hrefs=False)
+        )
 
     def publish_items(self, items: List[Item]):
         """Publish items."""