Switch object storage to use native AWS S3 SDK (#2260)

.gitignore

@@ -4,3 +4,4 @@
 .mypy_cache
 .venv
 venv
+.vscode

opencti-platform/opencti-graphql/config/default.json

@@ -112,6 +112,7 @@
   },
   "minio": {
     "bucket_name": "opencti-bucket",
+    "bucket_region": "us-east-1",
     "endpoint": "localhost",
     "port": 9000,
     "use_ssl": false,

opencti-platform/opencti-graphql/package.json

@@ -27,7 +27,10 @@
     "node": ">= 16.*"
   },
   "dependencies": {
-    "@aws-sdk/credential-providers": "3.110.0",
+    "@aws-sdk/client-s3": "^3.121.0",
+    "@aws-sdk/credential-provider-node": "^3.137.0",
+    "@aws-sdk/lib-storage": "^3.137.0",
+    "@aws-sdk/property-provider": "^3.127.0",
     "@elastic/elasticsearch": "7.17.0",
     "@graphql-tools/schema": "8.3.14",
     "@graphql-tools/utils": "8.6.13",
@@ -63,7 +66,6 @@
     "graphql-scalars": "1.17.0",
     "graphql-subscriptions": "2.0.0",
     "graphql-upload": "13.0.0",
-    "he": "1.2.0",
     "helmet": "5.1.0",
     "ioredis": "5.0.6",
     "iterall": "1.3.0",
@@ -73,7 +75,6 @@
     "merge-graphql-schemas": "1.7.8",
     "migrate": "1.8.0",
     "mime-types": "2.1.35",
-    "minio": "7.0.28",
     "moment": "2.29.3",
     "moment-range": "4.0.2",
     "nconf": "0.12.0",

opencti-platform/opencti-graphql/src/database/file-storage.js

@@ -0,0 +1,264 @@
+import * as s3 from '@aws-sdk/client-s3';
+import * as R from 'ramda';
+import { Upload } from '@aws-sdk/lib-storage';
+import { chain, CredentialsProviderError, memoize } from '@aws-sdk/property-provider';
+import { remoteProvider } from '@aws-sdk/credential-provider-node/dist-cjs/remoteProvider';
+import conf, { booleanConf, logApp, logAudit } from '../config/conf';
+import { now, sinceNowInMinutes } from '../utils/format';
+import { UPLOAD_ACTION } from '../config/audit';
+import { DatabaseError } from '../config/errors';
+import { deleteWorkForFile, loadExportWorksAsProgressFiles } from '../domain/work';
+import { buildPagination } from './utils';
+
+// Minio configuration
+const clientEndpoint = conf.get('minio:endpoint');
+const clientPort = conf.get('minio:port') || 9000;
+const clientAccessKey = conf.get('minio:access_key');
+const clientSecretKey = conf.get('minio:secret_key');
+const clientSessionToken = conf.get('minio:session_token');
+const bucketName = conf.get('minio:bucket_name') || 'opencti-bucket';
+const bucketRegion = conf.get('minio:bucket_region') || 'us-east-1';
+const excludedFiles = conf.get('minio:excluded_files') || ['.DS_Store'];
+const useSslConnection = booleanConf('minio:use_ssl', false);
+const useAwsRole = booleanConf('minio:use_aws_role', false);
+
+const credentialProvider = (init) => memoize(
+  chain(
+    async () => {
+      if (clientAccessKey && clientSecretKey && !useAwsRole) {
+        return {
+          accessKeyId: clientAccessKey,
+          secretAccessKey: clientSecretKey,
+          ...(clientSessionToken && { sessionToken: clientSessionToken })
+        };
+      }
+
+      throw new CredentialsProviderError('Unable to load credentials from OpenCTI config');
+    },
+    remoteProvider(init),
+    async () => {
+      throw new CredentialsProviderError('Could not load credentials from any providers', false);
+    }
+  ),
+  (credentials) => credentials.expiration !== undefined && credentials.expiration.getTime() - Date.now() < 300000,
+  (credentials) => credentials.expiration !== undefined
+);
+
+const s3Client = new s3.S3Client({
+  region: bucketRegion,
+  endpoint: getEndpoint(),
+  forcePathStyle: true,
+  credentialDefaultProvider: credentialProvider,
+  tls: useSslConnection
+});
+
+function getEndpoint() {
+  // If using AWS S3, unset the endpoint to let the library choose the best endpoint
+  if (clientEndpoint === 's3.amazonaws.com') {
+    return undefined;
+  }
+
+  return `${(useSslConnection ? 'https' : 'http')}://${clientEndpoint}:${clientPort}`;
+}
+
+export async function initializeBucket() {
+  try {
+    await s3Client.send(new s3.CreateBucketCommand({
+      Bucket: bucketName
+    }));
+    return true;
+  } catch (err) {
+    if (err instanceof s3.BucketAlreadyOwnedByYou) {
+      return true;
+    }
+
+    if (err instanceof s3.BucketAlreadyExists) {
+      throw new Error(`The S3 bucket name ${bucketName} is already in use, please choose another.`);
+    }
+
+    throw err;
+  }
+}
+
+export async function isStorageAlive() {
+  return initializeBucket();
+}
+
+export async function deleteFile(user, id) {
+  logApp.debug(`[FILE STORAGE] delete file ${id} by ${user.user_email}`);
+  await s3Client.send(new s3.DeleteObjectCommand({
+    Bucket: bucketName,
+    Key: id
+  }));
+  await deleteWorkForFile(user, id);
+  return true;
+}
+
+export async function deleteFiles(user, ids) {
+  logApp.debug(`[FILE STORAGE] delete files ${ids} by ${user.user_email}`);
+  for (let i = 0; i < ids.length; i += 1) {
+    const id = ids[i];
+    await deleteFile(user, id);
+  }
+  return true;
+}
+
+export async function downloadFile(id) {
+  try {
+    const object = await s3Client.send(new s3.GetObjectCommand({
+      Bucket: bucketName,
+      Key: id
+    }));
+    return object.Body;
+  } catch (err) {
+    logApp.info('[OPENCTI] Cannot retrieve file from S3', { error: err });
+    return null;
+  }
+}
+
+function streamToString(stream) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    stream.on('data', (chunk) => chunks.push(chunk));
+    stream.on('error', reject);
+    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+  });
+}
+
+export async function getFileContent(id) {
+  const object = await s3Client.send(new s3.GetObjectCommand({
+    Bucket: bucketName,
+    Key: id
+  }));
+
+  return streamToString(object.Body);
+}
+
+export function storeFileConverter(user, file) {
+  return {
+    id: file.id,
+    name: file.name,
+    version: file.metaData.version,
+    mime_type: file.metaData.mimetype,
+  };
+}
+
+export async function loadFile(user, filename) {
+  try {
+    const object = await s3Client.send(new s3.GetObjectCommand({
+      Bucket: bucketName,
+      Key: filename
+    }));
+
+    return {
+      id: filename,
+      name: decodeURIComponent(object.Metadata.filename || 'unknown'),
+      size: object.ContentLength,
+      information: '',
+      lastModified: object.LastModified,
+      lastModifiedSinceMin: sinceNowInMinutes(object.LastModified),
+      metaData: { ...object.Metadata, messages: [], errors: [] },
+      uploadStatus: 'complete'
+    };
+  } catch (err) {
+    if (err instanceof s3.NoSuchKey) {
+      throw DatabaseError('File not found', { user_id: user.id, filename });
+    }
+
+    throw err;
+  }
+}
+
+export function isFileObjectExcluded(id) {
+  const fileName = id.includes('/') ? R.last(id.split('/')) : id;
+  return excludedFiles.map((e) => e.toLowerCase()).includes(fileName.toLowerCase());
+}
+
+export async function rawFilesListing(user, directory, recursive = false) {
+  let truncated = true;
+  let pageMarker;
+  const objects = [];
+  const requestParams = {
+    Bucket: bucketName,
+    Prefix: directory || undefined,
+    Delimiter: recursive ? undefined : '/'
+  };
+
+  while (truncated) {
+    const response = await s3Client.send(new s3.ListObjectsV2Command(requestParams));
+
+    if (!response.Contents) {
+      truncated = false;
+      break;
+    }
+
+    objects.push(...response.Contents);
+    truncated = response.IsTruncated;
+    if (truncated) {
+      pageMarker = response.Contents.slice(-1)[0].Key;
+      requestParams.Marker = pageMarker;
+    }
+  }
+
+  return Promise.all(objects
+    .filter((obj) => !isFileObjectExcluded(obj.Key))
+    .map((obj) => loadFile(user, obj.Key)));
+}
+
+export async function upload(user, path, fileUpload, metadata = {}) {
+  const { createReadStream, filename, mimetype, encoding = '', version = now() } = await fileUpload;
+  const readStream = createReadStream();
+
+  logAudit.info(user, UPLOAD_ACTION, { path, filename, metadata });
+
+  const key = `${path}/${filename}`;
+  const fullMetadata = {
+    ...metadata,
+    filename: encodeURIComponent(filename),
+    mimetype,
+    encoding,
+    version
+  };
+
+  const s3Upload = new Upload({
+    client: s3Client,
+    params: {
+      Bucket: bucketName,
+      Key: key,
+      Body: readStream,
+      Metadata: fullMetadata
+    }
+  });
+  await s3Upload.done();
+
+  return {
+    id: key,
+    name: filename,
+    size: readStream.bytesRead,
+    information: '',
+    lastModified: new Date(),
+    lastModifiedSinceMin: sinceNowInMinutes(new Date()),
+    metaData: { ...fullMetadata, messages: [], errors: [] },
+    uploadStatus: 'complete'
+  };
+}
+
+export async function filesListing(user, first, path, entityId = null) {
+  const files = await rawFilesListing(user, path);
+  const inExport = await loadExportWorksAsProgressFiles(user, path);
+  const allFiles = R.concat(inExport, files);
+  const sortedFiles = R.sort((a, b) => b.lastModified - a.lastModified, allFiles);
+  let fileNodes = R.map((f) => ({ node: f }), sortedFiles);
+  if (entityId) {
+    fileNodes = R.filter((n) => n.node.metaData.entity_id === entityId, fileNodes);
+  }
+  return buildPagination(first, null, fileNodes, allFiles.length);
+}
+
+export async function deleteAllFiles(user, path) {
+  const files = await rawFilesListing(user, path);
+  const inExport = await loadExportWorksAsProgressFiles(user, path);
+  const allFiles = R.concat(inExport, files);
+  const ids = allFiles.map((file) => file.id);
+  return deleteFiles(user, ids);
+}

opencti-platform/opencti-graphql/src/database/middleware.js

@@ -199,7 +199,7 @@ import {
   yearFormat,
 } from '../utils/format';
 import { checkObservableSyntax } from '../utils/syntax';
-import { deleteAllFiles, storeFileConverter, upload } from './minio';
+import { deleteAllFiles, storeFileConverter, upload } from './file-storage';
 import { BYPASS, BYPASS_REFERENCE, filterElementsAccordingToUser, SYSTEM_USER } from '../utils/access';
 import { isRuleUser, RULE_MANAGER_USER, RULES_ATTRIBUTES_BEHAVIOR } from '../rules/rules';
 import {

opencti-platform/opencti-graphql/src/database/minio.js

@@ -1,243 +0,0 @@
-import * as Minio from 'minio';
-import * as He from 'he';
-import * as R from 'ramda';
-import querystring from 'querystring';
-import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
-import conf, { booleanConf, configureCA, logApp, logAudit } from '../config/conf';
-import { buildPagination } from './utils';
-import { loadExportWorksAsProgressFiles, deleteWorkForFile } from '../domain/work';
-import { now, sinceNowInMinutes } from '../utils/format';
-import { DatabaseError } from '../config/errors';
-import { UPLOAD_ACTION } from '../config/audit';
-
-// Minio configuration
-const clientEndpoint = conf.get('minio:endpoint');
-const clientPort = conf.get('minio:port') || 9000;
-const clientCA = conf.get('minio:ca');
-const clientAccessKey = conf.get('minio:access_key');
-const clientSecretKey = conf.get('minio:secret_key');
-const bucketName = conf.get('minio:bucket_name') || 'opencti-bucket';
-const bucketRegion = conf.get('minio:bucket_region') || 'us-east-1';
-const excludedFiles = conf.get('minio:excluded_files') || ['.DS_Store'];
-const useSslConnection = booleanConf('minio:use_ssl', false);
-const useAwsRole = booleanConf('minio:use_aws_role', false);
-
-// Credential global variable
-const minioCredentials = {
-  accessKey: String(clientAccessKey),
-  secretKey: String(clientSecretKey),
-  sessionToken: undefined,
-  expiration: -1
-};
-
-const getMinioClient = async () => {
-  // Attempt to fetch AWS role for authentication if enabled
-  if (useAwsRole) {
-    // Add 5 minutes to the current time so that new credentials are fetched
-    const expireDate = new Date((new Date()).getTime() + 300000);
-    const expirationTime = minioCredentials.expiration;
-    const isCredentialsExpired = expirationTime === -1 || (expirationTime && expirationTime < expireDate);
-    if (isCredentialsExpired) {
-      try {
-        const credentialProvider = fromNodeProviderChain();
-        const awsCredentials = await credentialProvider();
-        minioCredentials.accessKey = awsCredentials.accessKeyId;
-        minioCredentials.secretKey = awsCredentials.secretAccessKey;
-        minioCredentials.sessionToken = awsCredentials.sessionToken;
-        minioCredentials.expiration = awsCredentials.expiration;
-      } catch (e) {
-        logApp.error('[MINIO] Failed to fetch AWS role credentials', { error: e });
-      }
-    }
-  }
-  // Return the new client
-  return new Minio.Client({
-    endPoint: clientEndpoint,
-    port: clientPort,
-    useSSL: useSslConnection,
-    accessKey: minioCredentials.accessKey,
-    secretKey: minioCredentials.secretKey,
-    sessionToken: minioCredentials.sessionToken,
-    reqOptions: {
-      ...configureCA(clientCA),
-      servername: clientEndpoint,
-    },
-  });
-};
-
-export const initializeMinioBucket = async () => {
-  const minioClient = await getMinioClient();
-  return new Promise((resolve, reject) => {
-    try {
-      minioClient.bucketExists(bucketName, (existErr, exists) => {
-        if (existErr) {
-          reject(existErr);
-          return;
-        }
-        if (!exists) {
-          minioClient.makeBucket(bucketName, bucketRegion, (createErr) => {
-            if (createErr) reject(createErr);
-            resolve(true);
-          });
-        }
-        resolve(exists);
-      });
-    } catch (e) {
-      reject(e);
-    }
-  });
-};
-
-export const isStorageAlive = () => {
-  return initializeMinioBucket();
-};
-
-export const deleteFile = async (user, id) => {
-  const minioClient = await getMinioClient();
-  logApp.debug(`[MINIO] delete file ${id} by ${user.user_email}`);
-  await minioClient.removeObject(bucketName, id);
-  await deleteWorkForFile(user, id);
-  return true;
-};
-
-export const deleteFiles = async (user, ids) => {
-  logApp.debug(`[MINIO] delete files ${ids} by ${user.user_email}`);
-  for (let i = 0; i < ids.length; i += 1) {
-    const id = ids[i];
-    await deleteFile(user, id);
-  }
-  return true;
-};
-
-export const downloadFile = async (id) => {
-  const minioClient = await getMinioClient();
-  try {
-    return minioClient.getObject(bucketName, id);
-  } catch (err) {
-    logApp.info('[OPENCTI] Cannot retrieve file on MinIO', { error: err });
-    return null;
-  }
-};
-
-export const getFileContent = (id) => {
-  return new Promise((resolve, reject) => {
-    getMinioClient().then((minioClient) => {
-      let str = '';
-      minioClient.getObject(bucketName, id, (err, stream) => {
-        stream.on('data', (data) => {
-          str += data.toString('utf-8');
-        });
-        stream.on('end', () => {
-          resolve(str);
-        });
-        stream.on('error', (error) => {
-          reject(error);
-        });
-      });
-    }).catch((err) => {
-      reject(err);
-    });
-  });
-};
-
-export const storeFileConverter = (user, file) => {
-  return {
-    id: file.id,
-    name: file.name,
-    version: file.metaData.version,
-    mime_type: file.metaData.mimetype,
-  };
-};
-
-export const loadFile = async (user, filename) => {
-  const minioClient = await getMinioClient();
-
-  try {
-    const stat = await minioClient.statObject(bucketName, filename);
-    return {
-      id: filename,
-      name: querystring.unescape(stat.metaData.filename),
-      size: stat.size,
-      information: '',
-      lastModified: stat.lastModified,
-      lastModifiedSinceMin: sinceNowInMinutes(stat.lastModified),
-      metaData: { ...stat.metaData, messages: [], errors: [] },
-      uploadStatus: 'complete',
-    };
-  } catch (err) {
-    throw DatabaseError('File not found', { user_id: user.id, filename });
-  }
-};
-
-const isFileObjectExcluded = (id) => {
-  const fileName = id.includes('/') ? R.last(id.split('/')) : id;
-  return excludedFiles.map((e) => e.toLowerCase()).includes(fileName.toLowerCase());
-};
-export const rawFilesListing = (user, directory, recursive = false) => {
-  return new Promise((resolve, reject) => {
-    getMinioClient().then((minioClient) => {
-      const files = [];
-      const stream = minioClient.listObjectsV2(bucketName, directory, recursive);
-      stream.on('data', async (obj) => {
-        if (obj.name && !isFileObjectExcluded(obj.name)) {
-          files.push(R.assoc('id', obj.name, obj));
-        }
-      });
-      /* istanbul ignore next */
-      stream.on('error', (e) => {
-        logApp.error('[MINIO] Error listing files', { error: e });
-        reject(e);
-      });
-      stream.on('end', () => resolve(files));
-    }).catch((err) => {
-      reject(err);
-    });
-  }).then((files) => {
-    return Promise.all(
-      R.map((elem) => {
-        const filename = He.decode(elem.name);
-        return loadFile(user, filename);
-      }, files)
-    );
-  });
-};
-
-export const upload = async (user, path, fileUpload, metadata = {}) => {
-  const minioClient = await getMinioClient();
-  const { createReadStream, filename, mimetype, encoding = '', version = now() } = await fileUpload;
-  logAudit.info(user, UPLOAD_ACTION, { path, filename, metadata });
-  const escapeName = querystring.escape(filename);
-  const internalMeta = { filename: escapeName, mimetype, encoding, version };
-  const fileMeta = { ...metadata, ...internalMeta };
-  const fileDirName = `${path}/${filename}`;
-  logApp.debug(`[MINIO] Upload file ${fileDirName} by ${user.user_email}`);
-  // Upload the file in the storage
-  return new Promise((resolve, reject) => {
-    const fileStream = createReadStream();
-    minioClient.putObject(bucketName, fileDirName, fileStream, null, fileMeta, (err) => {
-      if (err) {
-        return reject(err);
-      }
-      return resolve(loadFile(user, fileDirName));
-    });
-  });
-};
-
-export const filesListing = async (user, first, path, entityId = null) => {
-  const files = await rawFilesListing(user, path);
-  const inExport = await loadExportWorksAsProgressFiles(user, path);
-  const allFiles = R.concat(inExport, files);
-  const sortedFiles = R.sort((a, b) => b.lastModified - a.lastModified, allFiles);
-  let fileNodes = R.map((f) => ({ node: f }), sortedFiles);
-  if (entityId) {
-    fileNodes = R.filter((n) => n.node.metaData.entity_id === entityId, fileNodes);
-  }
-  return buildPagination(first, null, fileNodes, allFiles.length);
-};
-
-export const deleteAllFiles = async (user, path) => {
-  const files = await rawFilesListing(user, path);
-  const inExport = await loadExportWorksAsProgressFiles(user, path);
-  const allFiles = R.concat(inExport, files);
-  return Promise.all(allFiles.map((file) => deleteFile(user, file.id)));
-};

opencti-platform/opencti-graphql/src/domain/file.js

@@ -1,5 +1,5 @@
 import * as R from 'ramda';
-import { loadFile, upload } from '../database/minio';
+import { loadFile, upload } from '../database/file-storage';
 import { pushToConnector } from '../database/rabbitmq';
 import { createWork } from './work';
 import { logApp } from '../config/conf';

opencti-platform/opencti-graphql/src/domain/stixCoreObject.js

@@ -45,7 +45,7 @@ import { createWork, workToExportFile } from './work';
 import { pushToConnector } from '../database/rabbitmq';
 import { now } from '../utils/format';
 import { ENTITY_TYPE_CONNECTOR } from '../schema/internalObject';
-import { deleteFile, loadFile, storeFileConverter, upload } from '../database/minio';
+import { deleteFile, loadFile, storeFileConverter, upload } from '../database/file-storage';
 import { elUpdateElement } from '../database/engine';
 import { getInstanceIds } from '../schema/identifier';
 import { askEntityExport } from './stix';

opencti-platform/opencti-graphql/src/domain/stixCoreRelationship.js

@@ -43,7 +43,7 @@ import {
 import { listRelations } from '../database/middleware-loader';
 import { askEntityExport, askListExport, exportTransformFilters } from './stix';
 import { workToExportFile } from './work';
-import { upload } from '../database/minio';
+import { upload } from '../database/file-storage';
 
 export const findAll = async (user, args) => {
   return listRelations(user, R.propOr(ABSTRACT_STIX_CORE_RELATIONSHIP, 'relationship_type', args), args);

opencti-platform/opencti-graphql/src/domain/stixCyberObservable.js

@@ -26,7 +26,7 @@ import { addIndicator } from './indicator';
 import { FunctionalError, UnsupportedError } from '../config/errors';
 import { createStixPattern } from '../python/pythonBridge';
 import { checkObservableSyntax } from '../utils/syntax';
-import { upload } from '../database/minio';
+import { upload } from '../database/file-storage';
 import {
   isStixCyberObservable,
   isStixCyberObservableHashedObservable,

opencti-platform/opencti-graphql/src/domain/stixDomainObject.js

@@ -16,7 +16,7 @@ import {
 } from '../database/middleware';
 import { listEntities } from '../database/middleware-loader';
 import { elCount } from '../database/engine';
-import { upload } from '../database/minio';
+import { upload } from '../database/file-storage';
 import { workToExportFile } from './work';
 import { FunctionalError, UnsupportedError } from '../config/errors';
 import { isEmptyField, READ_INDEX_STIX_DOMAIN_OBJECTS } from '../database/utils';

opencti-platform/opencti-graphql/src/http/httpPlatform.js

@@ -15,7 +15,7 @@ import contentDisposition from 'content-disposition';
 import { basePath, booleanConf, DEV_MODE, formatPath, logApp, logAudit } from '../config/conf';
 import passport, { empty, isStrategyActivated, STRATEGY_CERT } from '../config/providers';
 import { authenticateUser, authenticateUserFromRequest, loginFromProvider, userWithOrigin } from '../domain/user';
-import { downloadFile, getFileContent, loadFile } from '../database/minio';
+import { downloadFile, getFileContent, loadFile } from '../database/file-storage';
 import createSeeMiddleware from '../graphql/sseMiddleware';
 import initTaxiiApi from './httpTaxii';
 import { LOGIN_ACTION } from '../config/audit';

opencti-platform/opencti-graphql/src/initialization.js

@@ -4,7 +4,7 @@ import semver from 'semver';
 import { booleanConf, logApp, PLATFORM_VERSION } from './config/conf';
 import { elCreateIndexes, elIndexExists, searchEngineInit } from './database/engine';
 import { initializeAdminUser } from './config/providers';
-import { initializeMinioBucket, isStorageAlive } from './database/minio';
+import { initializeBucket, isStorageAlive } from './database/file-storage';
 import { rabbitMQIsAlive } from './database/rabbitmq';
 import { addMarkingDefinition } from './domain/markingDefinition';
 import { addSettings } from './domain/settings';
@@ -315,7 +315,7 @@ const platformInit = async (withMarkings = true) => {
     const alreadyExists = await isExistingPlatform();
     if (!alreadyExists) {
       logApp.info('[INIT] New platform detected, initialization...');
-      await initializeMinioBucket();
+      await initializeBucket();
       await initializeSchema();
       await initializeMigration();
       await initializeData(withMarkings);

opencti-platform/opencti-graphql/src/manager/syncManager.js

@@ -85,7 +85,7 @@ import { ENTITY_TYPE_SYNC } from '../schema/internalObject';
 import { createSyncHttpUri, httpBase, patchSync } from '../domain/connector';
 import { EVENT_CURRENT_VERSION, lockResource } from '../database/redis';
 import { stixCoreObjectImportDelete, stixCoreObjectImportPush } from '../domain/stixCoreObject';
-import { rawFilesListing } from '../database/minio';
+import { rawFilesListing } from '../database/file-storage';
 import { STIX_EXT_OCTI } from '../types/stix-extensions';
 import { utcDate } from '../utils/format';
 import { listEntities } from '../database/middleware-loader';

opencti-platform/opencti-graphql/src/migrations/1652125339035-files_database.js

@@ -1,6 +1,6 @@
 import * as R from 'ramda';
 import { Promise } from 'bluebird';
-import { deleteFiles, rawFilesListing, storeFileConverter } from '../database/minio';
+import { deleteFiles, rawFilesListing, storeFileConverter } from '../database/file-storage';
 import { SYSTEM_USER } from '../utils/access';
 import { elUpdate, ES_MAX_CONCURRENCY } from '../database/engine';
 import { internalLoadById } from '../database/middleware';

opencti-platform/opencti-graphql/src/resolvers/externalReference.js

@@ -18,7 +18,7 @@ import withCancel from '../graphql/subscriptionWrapper';
 import { RELATION_EXTERNAL_REFERENCE } from '../schema/stixMetaRelationship';
 import { buildRefRelationKey } from '../schema/general';
 import { worksForSource } from '../domain/work';
-import { filesListing } from '../database/minio';
+import { filesListing } from '../database/file-storage';
 import { stixCoreObjectImportPush } from '../domain/stixCoreObject';
 import { connectorsForEnrichment } from '../database/repository';
 

opencti-platform/opencti-graphql/src/resolvers/file.js

@@ -1,5 +1,5 @@
 import { GraphQLUpload } from 'graphql-upload';
-import { deleteFile, filesListing, loadFile } from '../database/minio';
+import { deleteFile, filesListing, loadFile } from '../database/file-storage';
 import { askJobImport, uploadImport, uploadPending } from '../domain/file';
 import { worksForSource } from '../domain/work';
 import { stixCoreObjectImportDelete } from '../domain/stixCoreObject';

opencti-platform/opencti-graphql/src/resolvers/stixCoreObject.js

@@ -26,7 +26,7 @@ import { creator } from '../domain/log';
 import { fetchEditContext, pubsub } from '../database/redis';
 import { batchLoader, stixLoadByIdStringify } from '../database/middleware';
 import { worksForSource } from '../domain/work';
-import { filesListing } from '../database/minio';
+import { filesListing } from '../database/file-storage';
 import { stixDomainObjectCleanContext, stixDomainObjectEditContext } from '../domain/stixDomainObject';
 import { BUS_TOPICS } from '../config/conf';
 import { ABSTRACT_STIX_CORE_OBJECT } from '../schema/general';

opencti-platform/opencti-graphql/src/resolvers/stixCoreRelationship.js

@@ -31,7 +31,7 @@ import { RELATION_CREATED_BY, RELATION_OBJECT_LABEL, RELATION_OBJECT_MARKING } f
 import { ABSTRACT_STIX_CORE_RELATIONSHIP, buildRefRelationKey } from '../schema/general';
 import { elBatchIds } from '../database/engine';
 import { findById as findStatusById, getTypeStatuses } from '../domain/status';
-import { filesListing } from '../database/minio';
+import { filesListing } from '../database/file-storage';
 
 const loadByIdLoader = batchLoader(elBatchIds);
 const createdByLoader = batchLoader(batchCreatedBy);

opencti-platform/opencti-graphql/src/resolvers/stixCyberObservable.js

@@ -27,7 +27,7 @@ import {
 import { pubsub } from '../database/redis';
 import withCancel from '../graphql/subscriptionWrapper';
 import { stixCoreObjectImportPush, stixCoreRelationships } from '../domain/stixCoreObject';
-import { filesListing } from '../database/minio';
+import { filesListing } from '../database/file-storage';
 import { ABSTRACT_STIX_CYBER_OBSERVABLE } from '../schema/general';
 import { stixHashesToInput } from '../schema/fieldDataAdapter';
 import { stixCyberObservableOptions } from '../schema/stixCyberObservable';

opencti-platform/opencti-graphql/src/resolvers/stixDomainObject.js

@@ -25,7 +25,7 @@ import {
 import { findById as findStatusById, getTypeStatuses } from '../domain/status';
 import { pubsub } from '../database/redis';
 import withCancel from '../graphql/subscriptionWrapper';
-import { filesListing } from '../database/minio';
+import { filesListing } from '../database/file-storage';
 import { ABSTRACT_STIX_DOMAIN_OBJECT } from '../schema/general';
 import { stixDomainObjectOptions } from '../schema/stixDomainObject';
 import { stixCoreObjectImportPush } from '../domain/stixCoreObject';

opencti-platform/opencti-graphql/tests/02-integration/01-database/file-storage-test.js

@@ -1,5 +1,5 @@
 import { head } from 'ramda';
-import { deleteFile, downloadFile, filesListing, loadFile } from '../../../src/database/minio';
+import { deleteFile, downloadFile, filesListing, loadFile } from '../../../src/database/file-storage';
 import { execPython3 } from '../../../src/python/pythonBridge';
 import { ADMIN_USER, API_TOKEN, API_URI, PYTHON_PATH } from '../../utils/testQuery';
 import { elLoadById } from '../../../src/database/engine';
@@ -19,7 +19,7 @@ const exportFileName = '(ExportFileStix)_Malware-Paradise Ransomware_all.json';
 const exportFileId = (malware) => `export/Malware/${malware.id}/${exportFileName}`;
 const importFileId = `import/global/${exportFileName}`;
 
-describe('Minio file listing', () => {
+describe('File storage file listing', () => {
   it('should file upload succeed', async () => {
     await startModules();
     const malware = await elLoadById(ADMIN_USER, 'malware--faa5b705-cf44-4e50-8472-29e5fec43c3c');