Aggregation improvements

ci/apiv2/test_task.py

@@ -15,7 +15,7 @@ def create_test_agent_object(self, *nargs, delete=True, **kwargs):
         dummy_agent.send_process(progress=50)
         dummy_agent.send_process(progress=100, state=ProcessState.EXHAUSTED)
         dummy_agent.get_chunk()
-        return Task.objects.get(taskId=retval['task'].id)
+        return Task.objects.params(**{"aggregate[task]": "totalNumberOfChunks"}).get(taskId=retval['task'].id)
 
     def create_test_object(self, **kwargs):
         hashlist_kwargs = kwargs.copy()

ci/phpunit/dba/AbstractModelFactoryTest.php

@@ -589,6 +589,49 @@ public function testColumnFilterSuccess(): void {
     $this->assertEquals([1, 125, 72], $column);
   }
 
+  /**
+   * Test receiving the column of a query with an order
+   *
+   * @return void
+   * @throws Exception
+   */
+  public function testColumnFilterSuccessOrdered(): void {
+    $testid = uniqid();
+    $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype1' . $testid, 1, 0));
+    $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype2' . $testid, 125, 0));
+    $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype3' . $testid, 72, 0));
+
+    $qF = new LikeFilter(HashType::DESCRIPTION, "%" . $testid);
+    $oF = new OrderFilter(HashType::IS_SALTED, "ASC");
+    $column = Factory::getHashTypeFactory()->columnFilter([Factory::FILTER => $qF, Factory::ORDER => $oF], HashType::IS_SALTED);
+    $this->assertEquals([1, 72, 125], $column);
+
+    $oF = new OrderFilter(HashType::IS_SALTED, "DESC");
+    $column = Factory::getHashTypeFactory()->columnFilter([Factory::FILTER => $qF, Factory::ORDER => $oF], HashType::IS_SALTED);
+    $this->assertEquals([125, 72, 1], $column);
+  }
+
+  /**
+   * Test querying multiple columns with the column filter
+   *
+   * @return void
+   * @throws Exception
+   */
+  public function testColumnFilterSuccessMultiple(): void {
+    $testid = uniqid();
+    $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype1' . $testid, 1, 0));
+    $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype2' . $testid, 125, 0));
+    $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype3' . $testid, 72, 1));
+
+    $qF = new LikeFilter(HashType::DESCRIPTION, "%" . $testid);
+    $columns = Factory::getHashTypeFactory()->columnFilter([Factory::FILTER => $qF], [HashType::IS_SALTED, HashType::IS_SLOW_HASH]);
+    $this->assertEquals([[1, 0], [125, 0], [72, 1]], $columns);
+
+    $oF = new OrderFilter(HashType::IS_SALTED, "ASC");
+    $columns = Factory::getHashTypeFactory()->columnFilter([Factory::FILTER => $qF, Factory::ORDER => $oF], [HashType::IS_SALTED, HashType::IS_SLOW_HASH]);
+    $this->assertEquals([[1, 0], [72, 1], [125, 0]], $columns);
+  }
+
   /**
    * Test receiving the column of a query on a mapped column
    *
@@ -1137,8 +1180,9 @@ public function testColumnFilter(): void {
     // hashlist 1 and 3 should be returned
     $this->assertSame([$hashlist_1->getId(), $hashlist_3->getId()], $ids);
 
-    $qF = new QueryFilter(Hashlist::CRACKED, 5000, ">");
-    $ids = Factory::getHashlistFactory()->columnFilter([Factory::FILTER => $qF, Factory::ORDER => $oF], Hashlist::HASHLIST_ID);
+    $qF1 = new QueryFilter(Hashlist::CRACKED, 5000, ">");
+    $qF2 = new LikeFilter(Hashlist::HASHLIST_NAME, "%" . $testid);
+    $ids = Factory::getHashlistFactory()->columnFilter([Factory::FILTER => [$qF1, $qF2], Factory::ORDER => $oF], Hashlist::HASHLIST_ID);
     $this->assertSame([], $ids);
   }
 

src/dba/AbstractModelFactory.php

@@ -486,13 +486,19 @@ public function multicolAggregationFilter(array $options, array $aggregations):
 
   /**
    * @param $options array options of query (filters and joins)
-   * @param $column string single column key which should be retrieved
+   * @param $columns array|string single column key or array of column keys which should be retrieved
    * @return array of the column entries returned from this query
    * @throws Exception
    */
-  public function columnFilter(array $options, string $column): array {
-    $query = "SELECT " . Util::createPrefixedString($this->getMappedModelTable(), [self::getMappedModelKey($this->getNullObject(), $column)]);
-    $query = $query . " FROM " . $this->getMappedModelTable();
+  public function columnFilter(array $options, array|string $columns): array {
+    if (!is_array($columns)) {
+      $columns = [$columns];
+    }
+    $elements = [];
+    foreach ($columns as $column) {
+      $elements[] = Util::createPrefixedString($this->getMappedModelTable(), [self::getMappedModelKey($this->getNullObject(), $column)]);
+    }
+    $query = "SELECT " . join(",", $elements) . " FROM " . $this->getMappedModelTable();
 
     $vals = array();
 
@@ -502,12 +508,18 @@ public function columnFilter(array $options, string $column): array {
     if (array_key_exists(Factory::FILTER, $options)) {
       $query .= $this->applyFilters($vals, $options[Factory::FILTER]);
     }
+    if (array_key_exists(Factory::ORDER, $options)) {
+      $query .= $this->applyOrder($options[Factory::ORDER]);
+    }
 
     $dbh = self::getDB();
     $stmt = $dbh->prepare($query);
     $stmt->execute($vals);
 
-    return $stmt->fetchAll(PDO::FETCH_COLUMN);
+    if (sizeof($elements) == 1) {
+      return $stmt->fetchAll(PDO::FETCH_COLUMN);
+    }
+    return $stmt->fetchAll(PDO::FETCH_NUM);
   }
 
   /**

src/inc/apiv2/common/AbstractBaseAPI.php

@@ -8,9 +8,7 @@
 use Hashtopolis\inc\apiv2\error\InternalError;
 use Hashtopolis\inc\apiv2\error\ResourceNotFoundError;
 use Hashtopolis\inc\utils\AccessControl;
-use Hashtopolis\inc\utils\AccessUtils;
 use Hashtopolis\inc\defines\DAccessControl;
-use Hashtopolis\inc\utils\HashlistUtils;
 use Hashtopolis\dba\JoinFilter;
 use Hashtopolis\inc\HTException;
 use JsonException;
@@ -183,17 +181,35 @@ protected function getUpdateHandlers($id, $current_user): array {
    *
    * Implementations should use $includedData to collect related resources that should be included
    * in the API response, such as related entities or additional data.
+   * @throws HttpError
    */
   public function aggregateData(object $object, array &$includedData = [], ?array $aggregateFieldsets = null): array {
-    return [];
+    $aggregatedData = [];
+
+    if (is_array($aggregateFieldsets)) {
+      $fieldsets = $this->getAggregateFieldsets();
+      foreach ($fieldsets as $name => $fieldset) {
+        if (array_key_exists($name, $aggregateFieldsets)) {
+          $aggregateFieldsets[$name] = explode(",", $aggregateFieldsets[$name]);
+          foreach($aggregateFieldsets[$name] as $field) {
+            if(!array_key_exists($field, $fieldset)) {
+              throw new HttpError("Invalid aggregation requested!");
+            }
+            $aggregatedData[$field] = $fieldset[$field]($object);
+          }
+        }
+      }
+    }
+    return $aggregatedData;
   }
-
+  
   /**
-   * Return supported aggregate fieldsets/options for this endpoint.
+   * Return supported aggregate fieldsets/options for this endpoint, providing a callback to call to actually retrieve
+   * this aggregation on a specific object. All callbacks expect one argument being the api object.
    *
    * Format:
    * [
-   *   'resourceKey' => ['option1', 'option2']
+   *   'resourceKey' => ['option1' => [class, function], 'option2' => [class, function]]
    * ]
    */
   public function getAggregateFieldsets(): array {
@@ -658,6 +674,7 @@ protected function obj2Array(object $obj): array {
    * Convert DB object JSON:API Resource Object
    * @throws NotFoundExceptionInterface
    * @throws ContainerExceptionInterface
+   * @throws HttpError
    */
   protected function obj2Resource(object $obj, array &$expandResult = [], ?array $sparseFieldsets = null, ?array $aggregateFieldsets = null): array {
     // Convert values to JSON supported types
@@ -702,11 +719,13 @@ protected function obj2Resource(object $obj, array &$expandResult = [], ?array $
 
     if ($this instanceof AbstractModelAPI && get_class($obj) !== $this->getDBAClass()) {
       $apiClassObject = new $apiClass($this->container);
-    } else {
+    }
+    else {
       // use instance of this when the object is of the dba class of this api endpoint.
       // This way its possible to set object attributes in the post to be used in the aggregateData function.
       $apiClassObject = $this;
     }
+
     $aggregatedData = $apiClassObject->aggregateData($obj, $expandResult, $aggregateFieldsets);
     $attributes = array_merge($attributes, $aggregatedData);
 
@@ -1124,9 +1143,9 @@ protected function getPrimaryKey(): string {
   function getFilters(Request $request): array {
     return $this->getQueryParameterFamily($request, 'filter');
   }
-
+  
   protected static function checkJoinExists(array $joins, string $modelName) {
-    foreach($joins as $join) {
+    foreach ($joins as $join) {
       if ($join->getOtherFactory()->getModelName() === $modelName) {
         return true;
       }
@@ -1154,13 +1173,13 @@ protected function makeFilter(array $filters, object $apiClass, array &$joinFilt
       $cast_key = $matches['key'] == 'id' ? array_column($features, 'alias', 'dbname')[$this->getPrimaryKey()] : $matches['key'];
       if (strpos($cast_key, ".")) {
         //When the key contains a "." it should be a relation in format: "task.taskname" where task is the relation.
-          $relationObject = $this->retrieveRelationKey($cast_key);
-          $factory = $relationObject->factory;
-          $cast_key = $relationObject->cast_key;
-          if (!self::checkJoinExists($joinFilters, $factory->getModelName())) {
-            $joinFilters[] = new JoinFilter($factory, $relationObject->joinKey, $relationObject->key);
-          }
-          $features = $relationObject->features_relation;
+        $relationObject = $this->retrieveRelationKey($cast_key);
+        $factory = $relationObject->factory;
+        $cast_key = $relationObject->cast_key;
+        if (!self::checkJoinExists($joinFilters, $factory->getModelName())) {
+          $joinFilters[] = new JoinFilter($factory, $relationObject->joinKey, $relationObject->key);
+        }
+        $features = $relationObject->features_relation;
       }
 
       if (!array_key_exists($cast_key, $features)) {
@@ -1259,7 +1278,7 @@ protected function makeFilter(array $filters, object $apiClass, array &$joinFilt
     }
     return $qFs;
   }
-
+  
   /**
    * Retrieves the relation from a sort/filter value. ex task.taskName when task is a relation for the current
    * Model endpoint. This works only for relations of 1 deep
@@ -1277,17 +1296,19 @@ protected function retrieveRelationKey(string $value): object {
         $key = $relations[$relationString]['key'];
         $features_relation = $relationFeatures;
         $value = $parts[1];
-        return (object) [
+        return (object)[
           "factory" => $factory,
           "joinKey" => $joinKey,
           "key" => $key,
           "features_relation" => $features_relation,
           "cast_key" => $value
         ];
-      } else {
+      }
+      else {
         throw new HttpError("Invalid relation: " . $relationString);
       }
-    } else {
+    }
+    else {
       throw new HttpForbidden("Invalid key, multiple '.' found in key, but only relationships of one deep is allowed");
     }
   }
@@ -1376,7 +1397,7 @@ protected function processExpands(
     $expandKeys = array_keys($expandResult);
     $diffs = array_diff($expandKeys, $expands);
     $expands = array_merge($expands, $diffs);
-
+    
     foreach ($expands as $expand) {
       if (!array_key_exists($object->getId(), $expandResult[$expand])) {
         continue;
@@ -1417,7 +1438,7 @@ protected function validatePermissions(string $permissions, array $required_perm
     else {
       $rightgroup_perms = json_decode($permissions, true);
     }
-
+    
     if ($aud === "user_hashtopolis") {
       // Validate if no undefined permissions are set in $acl_mapping for the legacy permissions
       assert(count(array_diff(array_keys($rightgroup_perms), array_keys(self::$acl_mapping))) == 0);
@@ -1428,7 +1449,8 @@ protected function validatePermissions(string $permissions, array $required_perm
           $user_available_perms = array_unique(array_merge($user_available_perms, self::$acl_mapping[$rightgroup_perm]));
         }
       };
-    } else {
+    }
+    else {
       $user_available_perms = array_keys($rightgroup_perms, true, true);
     }
 
@@ -1641,7 +1663,7 @@ static function createJsonResponse(array $data = [], array $links = [], array $i
    * Get single Resource
    */
   protected static function getOneResource(object $apiClass, object $object, Request $request, Response $response, int $statusCode = 200): Response {
-    $apiClass->preCommon($request);  
+    $apiClass->preCommon($request);
     $validExpandables = $apiClass->getExpandables();
     $expands = $apiClass->makeExpandables($request, $validExpandables);
 

src/inc/apiv2/common/AbstractModelAPI.php

@@ -772,7 +772,7 @@ public static function getManyResources(object $apiClass, Request $request, Resp
 
     // Convert objects to data resources 
     foreach ($objects as $object) {
-      // Create object  
+      // Create object
       $newObject = $apiClass->obj2Resource($object, $expandResult, $request->getQueryParams()['fields'] ?? null, $request->getQueryParams()['aggregate'] ?? null);
       $includedResources = $apiClass->processExpands($apiClass, $expands, $object, $expandResult, $includedResources, $request->getQueryParams()['fields'] ?? null, $request->getQueryParams()['aggregate'] ?? null);
 

src/inc/apiv2/common/openAPISchema.routes.php

@@ -699,8 +699,8 @@
               if (empty($options)) {
                 continue;
               }
-              $aggregateExamples["aggregate[" . $fieldset . "]"] = implode(",", $options);
-              $aggregateDescriptionParts[] = $fieldset . ": " . implode(", ", $options);
+              $aggregateExamples["aggregate[" . $fieldset . "]"] = implode(",", array_keys($options));
+              $aggregateDescriptionParts[] = $fieldset . ": " . implode(", ", array_keys($options));
             }
 
             if (!empty($aggregateExamples)) {

src/inc/apiv2/model/AgentAPI.php

@@ -2,6 +2,8 @@
 
 namespace Hashtopolis\inc\apiv2\model;
 
+use Exception;
+use Hashtopolis\dba\Aggregation;
 use Hashtopolis\inc\utils\AccessUtils;
 use Hashtopolis\inc\utils\AgentUtils;
 use Hashtopolis\inc\defines\DHashcatStatus;
@@ -45,27 +47,52 @@ protected function getUpdateHandlers($id, $current_user): array {
     ];
   }
 
+  public function getAggregateFieldsets(): array {
+    return [
+      'agent' => [
+        'crackingTime' => [$this, 'getAggregateCrackingTime'],
+      ]
+    ];
+  }
+
+  /**
+   * @param object $object
+   * @return int
+   * @throws Exception
+   */
+  protected function getAggregateCrackingTime(object $object): int {
+    // in order to make sense of the diff, we need to make sure that both values solve time and dispatch time are set (i.e. >0).
+    $qF1 = new QueryFilter(Chunk::AGENT_ID, $object->getId(), "=");
+    $qF2 = new QueryFilter(Chunk::SOLVE_TIME, 0, ">");
+    $qF3 = new QueryFilter(Chunk::DISPATCH_TIME, 0, ">");
+    $agg1 = new Aggregation(Chunk::SOLVE_TIME, Aggregation::SUM);
+    $agg2 = new Aggregation(Chunk::DISPATCH_TIME, Aggregation::SUM);
+    $results = Factory::getChunkFactory()->multicolAggregationFilter([Factory::FILTER => [$qF1, $qF2, $qF3]], [$agg1, $agg2]);
+    return $results[$agg1->getName()] - $results[$agg2->getName()];
+  }
+
   /**
    * Overridable function to aggregate data in the object. active chunk of agent is appended to
    * $included_data.
    *
    * @param object $object the agent object were data is aggregated from
-   * @param array &$included_data
+   * @param array &$includedData
    * @param array|null $aggregateFieldsets
    * @return array not used here
+   * @throws Exception
    */
-  function aggregateData(object $object, array &$included_data = [], ?array $aggregateFieldsets = null): array {
+  function aggregateData(object $object, array &$includedData = [], ?array $aggregateFieldsets = null): array {
     $agentId = $object->getId();
     $qFs = [];
     $qFs[] = new QueryFilter(Chunk::AGENT_ID, $agentId, "=");
     $qFs[] = new QueryFilter(Chunk::STATE, DHashcatStatus::RUNNING, "=");
 
     $active_chunk = Factory::getChunkFactory()->filter([Factory::FILTER => $qFs], true);
     if ($active_chunk !== NULL) {
-      $included_data["chunks"][$agentId] = [$active_chunk];
+      $includedData["chunks"][$agentId] = [$active_chunk];
     }
 
-    return [];
+    return parent::aggregateData($object, $includedData, $aggregateFieldsets);
   }
 
   protected function getSingleACL(User $user, object $object): bool {