Add analytics logs, Implements #125

application.properties

@@ -96,4 +96,8 @@ max_concurrent_dl = 5
 
 [example]
 query =
-target = 
+target =
+
+[analytics]
+enable_logging_runs = False
+logging_file =

src/dgenies/config_reader.py

@@ -415,3 +415,20 @@ class AppConfigReader:
             return self.reader.get("example", "target")
         except (NoOptionError, NoSectionError):
             return ""
+
+    def _get_analytics_enabled(self):
+        try:
+            return self.reader.get("analytics", "enable_logging_runs").lower() == "true"
+        except (NoOptionError, NoSectionError):
+            return False
+
+    def _get_analytics_log_file(self):
+        try:
+            log_file = self._replace_vars(self.reader.get("analytics", "logging_file"))
+            if (log_file == "" or not os.path.exists(os.path.dirname(log_file))) and self._get_analytics_enabled():
+                raise ValueError("Analytics log file not set or has not existing parent folder")
+            return log_file
+        except (NoOptionError, NoSectionError):
+            if self._get_analytics_enabled():
+                raise ValueError("Analytics log file not set")
+            return None

src/dgenies/database.py

@@ -124,7 +124,17 @@ if MODE == "webserver":
             self.save()
 
 
+    if config.analytics_enabled:
 
+        class Analytics(BaseModel):
+            data_created = DateTimeField()
+            target_length = IntegerField()
+            query_length = IntegerField(null=True)
+            mail_client = CharField()
+            batch_type = CharField(max_length=20)
+
+        if not Analytics.table_exists():
+            Analytics.create_table()
 
     if not Job.table_exists():
         Job.create_table()

src/dgenies/lib/job_manager.py

@@ -872,6 +872,22 @@ class JobManager:
         if MODE == "webserver" and self.config.send_mail_status:
             self.send_mail_post()
 
+    def _save_analytics_data(self):
+        from dgenies.database import Analytics
+        with Job.connect():
+            job = Job.get(Job.id_job == self.id_job)
+            target_size = os.path.getsize(self.target.get_path())
+            query_size = None
+            if self.query is not None:
+                query_size = os.path.getsize(self.query.get_path())
+            log = Analytics.create(
+                date_created=datetime.now(),
+                target_size=target_size,
+                query_size=query_size,
+                mail_client=job.email,
+                batch_type=job.batch_type)
+            log.save()
+
     def _after_start(self, success, error_set):
         with Job.connect():
             if success:
@@ -880,6 +896,8 @@ class JobManager:
                     job = Job.get(Job.id_job == self.id_job)
                     job.status = status
                     job.save()
+                    if self.config.analytics_enabled:
+                        self._save_analytics_data()
                 else:
                     self.set_status_standalone("waiting")
                     self.prepare_data_in_thread()