Creating the correct status by default + migration (#428)

Co-authored-by: Samuel Hassine <samuel@hassine.fr>

.gitignore

@@ -1,3 +1,5 @@
 .DS_Store
 .idea
 *.iml
+connectors
+venv

opencti-platform/opencti-front/src/private/components/reports/Reports.js

@@ -61,7 +61,7 @@ class Reports extends Component {
       createdByRef: {
         label: 'Author',
         width: '20%',
-        isSortable: true,
+        isSortable: false,
       },
       published: {
         label: 'Publication date',

opencti-platform/opencti-graphql/config/schema/opencti.graphql

@@ -2456,7 +2456,6 @@ enum ReportsOrdering {
   modified
   published
   object_status
-  createdByRef
   created_at
   updated_at
   tags

opencti-platform/opencti-graphql/src/database/elasticSearch.js

@@ -53,7 +53,7 @@ const virtualTypes = ['Identity', 'Email', 'File', 'Stix-Domain-Entity', 'Stix-D
 
 export const REL_INDEX_PREFIX = 'rel_';
 export const INDEX_STIX_OBSERVABLE = 'stix_observables';
-export const INDEX_STIX_ENTITIES = 'stix_domain_entities';
+export const INDEX_STIX_ENTITIES = 'stix_domain_entities_v2';
 export const INDEX_STIX_RELATIONS = 'stix_relations';
 export const INDEX_WORK_JOBS = 'work_jobs_index';
 export const PLATFORM_INDICES = [INDEX_STIX_ENTITIES, INDEX_STIX_RELATIONS, INDEX_STIX_OBSERVABLE, INDEX_WORK_JOBS];
@@ -81,6 +81,10 @@ export const elVersion = () => {
     .then(info => info.body.version.number)
     .catch(() => 'Disconnected');
 };
+export const elIndexExists = async indexName => {
+  const existIndex = await el.indices.exists({ index: indexName });
+  return existIndex.body === true;
+};
 export const elCreateIndexes = async () => {
   return Promise.all(
     PLATFORM_INDICES.map(index => {
@@ -153,6 +157,9 @@ export const elCreateIndexes = async () => {
                     type: 'date',
                     format: 'strict_year_month',
                     ignore_malformed: true
+                  },
+                  object_status: {
+                    type: 'integer'
                   }
                 }
               }
@@ -681,9 +688,9 @@ export const elLoadByGraknId = (id, relationsMap, indices = PLATFORM_INDICES) =>
 export const elBulk = async args => {
   return el.bulk(args);
 };
-export const elReindex = async indexMaps => {
+export const elReindex = async indices => {
   return Promise.all(
-    indexMaps.map(indexMap => {
+    indices.map(indexMap => {
       return el
         .reindex({
           timeout: '60m',

opencti-platform/opencti-graphql/src/database/migration.js

@@ -92,32 +92,35 @@ const graknStateStorage = {
 const applyMigration = () => {
   logger.info('[MIGRATION] > Starting migration process');
   const set = new MigrationSet(graknStateStorage);
-  return graknStateStorage.load((err, state) => {
-    if (err) throw new Error(err);
-    // Set last run date on the set
-    set.lastRun = state.lastRun || null;
-    // Read migrations from webpack
-    const migrationSet = retrieveMigrations();
-    const stateMigrations = new Map(state.migrations ? state.migrations.map(i => [i.title, i]) : null);
-    for (let index = 0; index < migrationSet.length; index += 1) {
-      const migSet = migrationSet[index];
-      const migration = new Migration(migSet.title, migSet.up, migSet.down);
-      // Add timestamp if already done in remote state
-      const stateMigration = stateMigrations.get(migration.title);
-      if (stateMigration) {
-        migration.timestamp = stateMigration.timestamp;
-      } else {
-        logger.info(`[MIGRATION] > ${migSet.title} will be executed`);
+  return new Promise((resolve, reject) => {
+    graknStateStorage.load((err, state) => {
+      if (err) throw new Error(err);
+      // Set last run date on the set
+      set.lastRun = state.lastRun || null;
+      // Read migrations from webpack
+      const migrationSet = retrieveMigrations();
+      const stateMigrations = new Map(state.migrations ? state.migrations.map(i => [i.title, i]) : null);
+      for (let index = 0; index < migrationSet.length; index += 1) {
+        const migSet = migrationSet[index];
+        const migration = new Migration(migSet.title, migSet.up, migSet.down);
+        // Add timestamp if already done in remote state
+        const stateMigration = stateMigrations.get(migration.title);
+        if (stateMigration) {
+          migration.timestamp = stateMigration.timestamp;
+        } else {
+          logger.info(`[MIGRATION] > ${migSet.title} will be executed`);
+        }
+        set.addMigration(migration);
       }
-      set.addMigration(migration);
-    }
-    // Start the set migration
-    return set.up(migrationError => {
-      if (migrationError) {
-        logger.error('[GRAKN] Error during migration');
-        throw new Error(migrationError);
-      }
-      logger.info('[MIGRATION] > Migrations completed');
+      // Start the set migration
+      set.up(migrationError => {
+        if (migrationError) {
+          logger.error('[GRAKN] Error during migration');
+          reject(migrationError);
+        }
+        logger.info('[MIGRATION] > Migrations completed');
+        resolve();
+      });
     });
   });
 };

opencti-platform/opencti-graphql/src/domain/report.js

@@ -20,6 +20,11 @@ import { buildPagination } from '../database/utils';
 import { findAll as findAllStixObservables } from './stixObservable';
 import { findAll as findAllStixDomainEntities } from './stixDomainEntity';
 
+export const STATUS_STATUS_NEW = 0;
+export const STATUS_STATUS_PROGRESS = 1;
+export const STATUS_STATUS_ANALYZED = 2;
+export const STATUS_STATUS_CLOSED = 3;
+
 export const findById = reportId => {
   if (reportId.match(/[a-z-]+--[\w-]{36}/g)) {
     return loadEntityByStixId(reportId);
@@ -144,7 +149,9 @@ export const reportsDistributionByEntity = async args => {
 
 // region mutations
 export const addReport = async (user, report) => {
-  const created = await createEntity(report, 'Report');
+  // If no status in creation, just force STATUS_NEW
+  const reportWithStatus = report.object_status ? report : assoc('object_status', STATUS_STATUS_NEW, report);
+  const created = await createEntity(reportWithStatus, 'Report');
   return notify(BUS_TOPICS.StixDomainEntity.ADDED_TOPIC, created, user);
 };
 // endregion

opencti-platform/opencti-graphql/src/migrations/1579457088001-object_status_reindex.js

@@ -0,0 +1,15 @@
+import { elDeleteIndexes, elIndexExists, elReindex, INDEX_STIX_ENTITIES } from '../database/elasticSearch';
+
+export const up = async next => {
+  const applyMigration = await elIndexExists('stix_domain_entities');
+  if (applyMigration) {
+    await elReindex([{ source: 'stix_domain_entities', dest: INDEX_STIX_ENTITIES }]);
+    await elDeleteIndexes(['stix_domain_entities']);
+  }
+  next();
+};
+
+export const down = async next => {
+  // Reverting this cannot be done because we define the schema directly with the code...
+  next();
+};

opencti-platform/opencti-graphql/src/resolvers/report.js

@@ -49,7 +49,6 @@ const reportResolvers = {
     }
   },
   ReportsOrdering: {
-    createdByRef: `${REL_INDEX_PREFIX}created_by_ref.name`,
     markingDefinitions: `${REL_INDEX_PREFIX}object_marking_refs.definition`,
     tags: `${REL_INDEX_PREFIX}tagged.value`
   },