mirror of
https://github.com/OpenCTI-Platform/opencti.git
synced 2025-12-22 08:17:08 +00:00
[backend] make poster filtering with OR work (#12701)
This commit is contained in:
Jeremy Cloarec
@@ -139,7 +139,7 @@ import {
|
||||
RELATION_TYPE_SUBFILTER,
|
||||
TYPE_FILTER,
|
||||
} from '../utils/filtering/filtering-constants';
|
||||
import { type FilterGroup, FilterMode, FilterOperator } from '../generated/graphql';
|
||||
import { type Filter, type FilterGroup, FilterMode, FilterOperator } from '../generated/graphql';
|
||||
import {
|
||||
type AttributeDefinition,
|
||||
authorizedMembers,
|
||||
@@ -173,7 +173,7 @@ import { ENTITY_IPV4_ADDR, ENTITY_IPV6_ADDR, isStixCyberObservable } from '../sc
|
||||
import { lockResources } from '../lock/master-lock';
|
||||
import { DRAFT_OPERATION_CREATE, DRAFT_OPERATION_DELETE, DRAFT_OPERATION_DELETE_LINKED, DRAFT_OPERATION_UPDATE_LINKED } from '../modules/draftWorkspace/draftOperations';
|
||||
import { RELATION_SAMPLE } from '../modules/malwareAnalysis/malwareAnalysis-types';
|
||||
import { asyncFilter, asyncMap } from '../utils/data-processing';
|
||||
import { asyncMap } from '../utils/data-processing';
|
||||
import { doYield } from '../utils/eventloop-utils';
|
||||
import { RELATION_COVERED } from '../modules/securityCoverage/securityCoverage-types';
|
||||
import type { AuthContext, AuthUser } from '../types/user';
|
||||
@@ -191,7 +191,7 @@ import type {
|
||||
StoreRelation,
|
||||
} from '../types/store';
|
||||
import type { BasicStoreSettings } from '../types/settings';
|
||||
import { completeSpecialFilterKeys } from '../utils/filtering/filtering-completeSpecialFilterKeys';
|
||||
import { completeSpecialFilterKeys, type TaggedFilter, type TaggedFilterGroup } from '../utils/filtering/filtering-completeSpecialFilterKeys';
|
||||
import { IDS_ATTRIBUTES } from '../domain/attribute-utils';
|
||||
|
||||
const ELK_ENGINE = 'elk';
|
||||
@@ -621,7 +621,6 @@ export const buildDataRestrictions = async (
|
||||
opts: { includeAuthorities?: boolean | null } | null | undefined = {},
|
||||
): Promise<{ must: any[]; must_not: any[] }> => {
|
||||
const must: any[] = [];
|
||||
|
||||
const must_not: any[] = [];
|
||||
// If internal users of the system, we cancel rights checking
|
||||
if (INTERNAL_USERS[user.id]) {
|
||||
@@ -2542,6 +2541,70 @@ const buildSubQueryForFilterGroup = (
|
||||
}
|
||||
return null;
|
||||
};
|
||||
const buildSubQueryForTaggedFilterGroup = (
|
||||
context: AuthContext,
|
||||
user: AuthUser,
|
||||
inputFilters: any,
|
||||
): { subQuery: any; postFiltersTags: Set<string> } => {
|
||||
const { mode = 'and', filters = [], filterGroups = [] } = inputFilters;
|
||||
const localMustFilters: any = [];
|
||||
const localSubQueries: { subQuery: any; associatedTags: Set<string> }[] = [];
|
||||
const localPostFilterTags: Set<string> = new Set<string>();
|
||||
// Handle filterGroups
|
||||
for (let index = 0; index < filterGroups.length; index += 1) {
|
||||
const group = filterGroups[index];
|
||||
if (isFilterGroupNotEmpty(group)) {
|
||||
const { subQuery, postFiltersTags } = buildSubQueryForTaggedFilterGroup(context, user, group);
|
||||
if (subQuery) { // can be null
|
||||
localSubQueries.push({ subQuery, associatedTags: postFiltersTags });
|
||||
}
|
||||
if (postFiltersTags) {
|
||||
postFiltersTags.forEach((t: string) => localPostFilterTags.add(t));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Handle filters
|
||||
for (let index = 0; index < filters.length; index += 1) {
|
||||
const filter = filters[index];
|
||||
const isValidFilter = filter?.values || filter?.nested?.length > 0;
|
||||
if (isValidFilter) {
|
||||
const localMustFilter = buildLocalMustFilter(filter);
|
||||
if (filter?.postFilteringTag) {
|
||||
const associatedTag = filter?.postFilteringTag as string;
|
||||
localPostFilterTags.add(associatedTag);
|
||||
const associatedTags = new Set(associatedTag);
|
||||
localSubQueries.push({ subQuery: localMustFilter, associatedTags });
|
||||
} else {
|
||||
localSubQueries.push({ subQuery: localMustFilter, associatedTags: new Set<string>() });
|
||||
}
|
||||
}
|
||||
}
|
||||
// Wrap every subquery in a bool must tagged with name equal to local filter tags
|
||||
for (let i = 0; i < localSubQueries.length; i++) {
|
||||
const { subQuery, associatedTags } = localSubQueries[i];
|
||||
const tagsToApply = Array.from(localPostFilterTags).filter((t: string) => !associatedTags.has(t));
|
||||
const localMustFilter: any = {
|
||||
bool: {
|
||||
must: [subQuery],
|
||||
},
|
||||
};
|
||||
if (mode === 'or' && tagsToApply.length > 0) {
|
||||
localMustFilter.bool['_name'] = tagsToApply.join(POST_FILTER_TAG_SEPARATOR);
|
||||
}
|
||||
localMustFilters.push(localMustFilter);
|
||||
}
|
||||
|
||||
if (localMustFilters.length > 0) {
|
||||
const currentSubQuery = {
|
||||
bool: {
|
||||
should: localMustFilters,
|
||||
minimum_should_match: mode === 'or' ? 1 : localMustFilters.length,
|
||||
},
|
||||
};
|
||||
return { subQuery: currentSubQuery, postFiltersTags: localPostFilterTags };
|
||||
}
|
||||
return { subQuery: null, postFiltersTags: localPostFilterTags };
|
||||
};
|
||||
const getRuntimeEntities = async (context: AuthContext, user: AuthUser, entityType: string) => {
|
||||
const elements = await elPaginate<BasicStoreEntity>(context, user, READ_INDEX_STIX_DOMAIN_OBJECTS, {
|
||||
types: [entityType],
|
||||
@@ -2734,12 +2797,13 @@ type QueryBodyBuilderOpts = ProcessSearchArgs & BuildDraftFilterOpts & {
|
||||
first?: number | null;
|
||||
types?: string[] | null;
|
||||
search?: string | null;
|
||||
filters?: FilterGroup | null;
|
||||
filters?: TaggedFilterGroup | FilterGroup | null;
|
||||
noFiltersChecking?: boolean;
|
||||
startDate?: any;
|
||||
endDate?: any;
|
||||
dateAttribute?: string | null;
|
||||
includeAuthorities?: boolean | null;
|
||||
handlePostFiltering?: boolean;
|
||||
};
|
||||
const elQueryBodyBuilder = async (context: AuthContext, user: AuthUser, options: QueryBodyBuilderOpts) => {
|
||||
const {
|
||||
@@ -2761,6 +2825,7 @@ const elQueryBodyBuilder = async (context: AuthContext, user: AuthUser, options:
|
||||
endDate = null,
|
||||
dateAttribute = null,
|
||||
includeAuthorities = false,
|
||||
handlePostFiltering = false,
|
||||
} = options;
|
||||
const elFindByIdsToMap = async (c: AuthContext, u: AuthUser, i: string[], o: any) => {
|
||||
return elFindByIds<BasicStoreObject>(c, u, i, { ...o, toMap: true }) as Promise<Record<string, BasicStoreObject>>;
|
||||
@@ -2797,9 +2862,16 @@ const elQueryBodyBuilder = async (context: AuthContext, user: AuthUser, options:
|
||||
// Handle filters
|
||||
if (isFilterGroupNotEmpty(completeFilters)) {
|
||||
const finalFilters = await completeSpecialFilterKeys(context, user, completeFilters);
|
||||
const filtersSubQuery = buildSubQueryForFilterGroup(context, user, finalFilters);
|
||||
if (filtersSubQuery) {
|
||||
mustFilters.push(filtersSubQuery);
|
||||
if (handlePostFiltering) {
|
||||
const { subQuery: filtersSubQuery } = buildSubQueryForTaggedFilterGroup(context, user, finalFilters);
|
||||
if (filtersSubQuery) {
|
||||
mustFilters.push(filtersSubQuery);
|
||||
}
|
||||
} else {
|
||||
const filtersSubQuery = buildSubQueryForFilterGroup(context, user, finalFilters);
|
||||
if (filtersSubQuery) {
|
||||
mustFilters.push(filtersSubQuery);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Handle search
|
||||
@@ -2899,6 +2971,106 @@ const buildSearchResult = <T extends BasicStoreBase>(
|
||||
}
|
||||
return elements;
|
||||
};
|
||||
const POST_FILTER_TAG_SEPARATOR = ';';
|
||||
type PostFiltersTagMaps = {
|
||||
tagToFilterMap: Map<string, Filter>; // Is used during hit processing to know which post filtering application to exclude based on tag
|
||||
filterToTagMap: Map<Filter, string>; // Is used during applyTagToFilters to get tag to apply based on filter
|
||||
};
|
||||
const buildPostFiltersTagMaps = (
|
||||
filters: FilterGroup | undefined | null,
|
||||
): PostFiltersTagMaps => {
|
||||
const tagToFilterMap = new Map<string, Filter>();
|
||||
const filterToTagMap = new Map<Filter, string>();
|
||||
const result = { tagToFilterMap, filterToTagMap };
|
||||
if (isNotEmptyField(filters)) {
|
||||
const definedFilters = filters as FilterGroup;
|
||||
const postFilteringFilters = extractFiltersFromGroup(definedFilters, [INSTANCE_REGARDING_OF, INSTANCE_DYNAMIC_REGARDING_OF])
|
||||
.filter((filter) => isEmptyField(filter.operator) || filter.operator === 'eq');
|
||||
for (let i = 0; i < postFilteringFilters.length; i++) {
|
||||
const currentPostFilter: Filter = postFilteringFilters[i];
|
||||
tagToFilterMap.set(String(i), currentPostFilter);
|
||||
filterToTagMap.set(currentPostFilter, String(i));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
const applyTagToFilters = (
|
||||
filters: FilterGroup | undefined | null,
|
||||
postFiltersTagsMaps: PostFiltersTagMaps,
|
||||
): TaggedFilterGroup | undefined | null => {
|
||||
if (!filters || postFiltersTagsMaps.filterToTagMap.size <= 0) {
|
||||
return filters;
|
||||
}
|
||||
let newFilterGroups: TaggedFilterGroup[] = filters.filterGroups;
|
||||
if (newFilterGroups.length > 0) {
|
||||
newFilterGroups = [];
|
||||
for (let i = 0; i < filters.filterGroups.length; i++) {
|
||||
const filterGroupToTag = filters.filterGroups[i];
|
||||
const taggedFilterGroup = applyTagToFilters(filterGroupToTag, postFiltersTagsMaps);
|
||||
if (taggedFilterGroup) {
|
||||
newFilterGroups.push(taggedFilterGroup);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let newFilters: TaggedFilter[] = filters.filters;
|
||||
if (newFilters.length > 0) {
|
||||
newFilters = [];
|
||||
for (let j = 0; j < filters.filters.length; j++) {
|
||||
const currentFilter = filters.filters[j];
|
||||
if (postFiltersTagsMaps.filterToTagMap.has(currentFilter)) {
|
||||
const filterTag = postFiltersTagsMaps.filterToTagMap.get(currentFilter) as string;
|
||||
const taggedFilter = { ...currentFilter, postFilteringTag: filterTag };
|
||||
newFilters.push(taggedFilter);
|
||||
} else {
|
||||
newFilters.push(currentFilter);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
mode: filters.mode,
|
||||
filters: newFilters,
|
||||
filterGroups: newFilterGroups,
|
||||
};
|
||||
};
|
||||
const applyPostFilteringToElements = async <T extends BasicStoreBase>(
|
||||
context: AuthContext,
|
||||
user: AuthUser,
|
||||
elements: { element: T; tagsToIgnoreSet: Set<string> }[],
|
||||
postFiltersMaps?: PostFiltersTagMaps,
|
||||
): Promise<T[]> => {
|
||||
let filteredElements: T[] = [];
|
||||
const rawElements = elements.map((e) => e.element);
|
||||
if (elements.length <= 0 || !postFiltersMaps || postFiltersMaps.filterToTagMap.size <= 0) {
|
||||
filteredElements = rawElements;
|
||||
} else {
|
||||
const allPostFilteringFilters = Array.from(postFiltersMaps.filterToTagMap.entries());
|
||||
const tagToPostFilterMap = new Map<string, any>();
|
||||
for (let i = 0; i < allPostFilteringFilters.length; i++) {
|
||||
const [currentPostFilter, tag] = allPostFilteringFilters[i];
|
||||
const postFilterCheckFunction = await buildRegardingOfFilter<T>(context, user, rawElements, currentPostFilter);
|
||||
tagToPostFilterMap.set(tag, postFilterCheckFunction);
|
||||
}
|
||||
const allTags = Array.from(tagToPostFilterMap.keys());
|
||||
for (let j = 0; j < elements.length; j++) {
|
||||
const { element: currentElement, tagsToIgnoreSet } = elements[j];
|
||||
let keepCurrentElement = true;
|
||||
const postFilterTagsToIgnore: string[] = Array.from(tagsToIgnoreSet);
|
||||
const tagsToApply = allTags.filter((t) => !postFilterTagsToIgnore.includes(t));
|
||||
for (let k = 0; k < tagsToApply.length; k++) {
|
||||
const tagToApply = tagsToApply[k];
|
||||
const filterCheckToApply = tagToPostFilterMap.get(tagToApply);
|
||||
keepCurrentElement = keepCurrentElement && filterCheckToApply(currentElement);
|
||||
}
|
||||
if (keepCurrentElement) {
|
||||
filteredElements.push(currentElement);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return filteredElements;
|
||||
};
|
||||
export type PaginateOpts = QueryBodyBuilderOpts & {
|
||||
baseData?: boolean;
|
||||
baseFields?: string[];
|
||||
@@ -2933,7 +3105,9 @@ export const elPaginate = async <T extends BasicStoreBase>(
|
||||
filters,
|
||||
connectionFormat = true,
|
||||
} = options;
|
||||
const body = await elQueryBodyBuilder(context, user, options);
|
||||
const postFiltersMaps: PostFiltersTagMaps = buildPostFiltersTagMaps(filters);
|
||||
const tagAppliedFilters = applyTagToFilters(filters, postFiltersMaps);
|
||||
const body = await elQueryBodyBuilder(context, user, { ...options, filters: tagAppliedFilters, handlePostFiltering: true });
|
||||
if (body.size > ES_MAX_PAGINATION && !bypassSizeLimit) {
|
||||
logApp.info('[SEARCH] Pagination limited to max result config', { size: body.size, max: ES_MAX_PAGINATION });
|
||||
body.size = ES_MAX_PAGINATION;
|
||||
@@ -2955,12 +3129,30 @@ export const elPaginate = async <T extends BasicStoreBase>(
|
||||
const data = await elRawSearch(context, user, types !== null ? types : 'Any', query);
|
||||
const globalCount = data.hits.total.value;
|
||||
const elements = await elConvertHits<T>(data.hits.hits);
|
||||
// If filters contains an "in regards of" filter a post-security filtering is needed
|
||||
|
||||
const regardingOfFilter = elements.length === 0 ? undefined : await buildRegardingOfFilter<T>(context, user, elements, filters);
|
||||
const filteredElements = regardingOfFilter ? await asyncFilter(elements, regardingOfFilter) : elements;
|
||||
const filterCount = elements.length - filteredElements.length;
|
||||
const result = buildSearchResult(filteredElements, first, body.search_after, globalCount, filterCount, connectionFormat);
|
||||
let finalElements = elements;
|
||||
if (finalElements.length > 0 && postFiltersMaps.filterToTagMap.size > 0) {
|
||||
const elementsWithTags: { element: T; tagsToIgnoreSet: Set<string> }[] = [];
|
||||
for (let i = 0; i < data.hits.hits.length; i++) {
|
||||
const element = elements[i];
|
||||
const dataHit = data.hits.hits[i];
|
||||
const tagsToIgnoreSet = new Set<string>();
|
||||
if (dataHit.matched_queries) {
|
||||
for (let j = 0; j < dataHit.matched_queries.length; j++) {
|
||||
const currentMatchedQuery = dataHit.matched_queries[j];
|
||||
const matchedTags = currentMatchedQuery.split(POST_FILTER_TAG_SEPARATOR);
|
||||
for (let k = 0; k < matchedTags.length; k++) {
|
||||
const matchedTag = matchedTags[k];
|
||||
tagsToIgnoreSet.add(matchedTag);
|
||||
}
|
||||
}
|
||||
}
|
||||
elementsWithTags.push({ element, tagsToIgnoreSet });
|
||||
}
|
||||
// Since filters contains filters requiring post filterting (regardingOf, dynamicRegardingOf), a post-security filtering is needed
|
||||
finalElements = await applyPostFilteringToElements(context, user, elementsWithTags, postFiltersMaps);
|
||||
}
|
||||
const filterCount = elements.length - finalElements.length;
|
||||
const result = buildSearchResult(finalElements, first, body.search_after, globalCount, filterCount, connectionFormat);
|
||||
if (withResultMeta) {
|
||||
const lastProcessedSort = R.last(elements)?.sort;
|
||||
const endCursor = lastProcessedSort ? offsetToCursor(lastProcessedSort) : null;
|
||||
@@ -3485,98 +3677,87 @@ const buildRegardingOfFilter = async <T extends BasicStoreBase> (
|
||||
context: AuthContext,
|
||||
user: AuthUser,
|
||||
elements: T[],
|
||||
filters: FilterGroup | undefined | null,
|
||||
filter: Filter,
|
||||
) => {
|
||||
// First check if there is an "in regards of" filter
|
||||
// If its case we need to ensure elements are filtered according to denormalization rights.
|
||||
if (isNotEmptyField(filters)) {
|
||||
const definedFilters = filters as FilterGroup;
|
||||
const extractedFilters = extractFiltersFromGroup(definedFilters, [INSTANCE_REGARDING_OF, INSTANCE_DYNAMIC_REGARDING_OF])
|
||||
.filter((filter) => isEmptyField(filter.operator) || filter.operator === 'eq');
|
||||
if (extractedFilters.length > 0) {
|
||||
const targetValidatedIds = new Set();
|
||||
const sideIdManualInferred = new Map();
|
||||
for (let i = 0; i < extractedFilters.length; i += 1) {
|
||||
const { values } = extractedFilters[i];
|
||||
const ids = values.filter((v) => v.key === ID_SUBFILTER).map((f) => f.values).flat();
|
||||
const types = values.filter((v) => v.key === RELATION_TYPE_SUBFILTER).map((f) => f.values).flat();
|
||||
const inferredParameterValues = values.filter((v) => v.key === RELATION_INFERRED_SUBFILTER).map((f) => f.values).flat();
|
||||
const directionForced = R.head(values.filter((v) => v.key === INSTANCE_REGARDING_OF_DIRECTION_FORCED).map((f) => f.values).flat()) ?? false;
|
||||
const directionReverse = R.head(values.filter((v) => v.key === INSTANCE_REGARDING_OF_DIRECTION_REVERSE).map((f) => f.values).flat()) ?? false;
|
||||
// resolve all relationships that target the id values, forcing the type is available
|
||||
const paginateArgs: RepaginateOpts<BasicStoreRelation> = { baseData: true, types };
|
||||
const elementIds = elements.map(({ id }) => id);
|
||||
if (directionForced) {
|
||||
// If a direction is forced, build the filter in the correct direction
|
||||
const directedFilters = [];
|
||||
if (directionReverse) {
|
||||
directedFilters.push({ key: ['fromId'], values: elementIds });
|
||||
if (ids.length > 0) { // Ids can be empty if nothing configured by the user
|
||||
directedFilters.push({ key: ['toId'], values: ids });
|
||||
}
|
||||
} else {
|
||||
directedFilters.push({ key: ['toId'], values: elementIds });
|
||||
if (ids.length > 0) { // Ids can be empty if nothing configured by the user
|
||||
directedFilters.push({ key: ['fromId'], values: ids });
|
||||
}
|
||||
}
|
||||
paginateArgs.filters = { mode: FilterMode.And, filters: directedFilters, filterGroups: [] };
|
||||
} else {
|
||||
// If no direction is setup, create the filter group for both directions
|
||||
const filterTo = [{ key: ['fromId'], values: elementIds }];
|
||||
const filterFrom = [{ key: ['toId'], values: elementIds }];
|
||||
if (ids.length > 0) { // Ids can be empty if nothing configured by the user
|
||||
filterTo.push({ key: ['toId'], values: ids });
|
||||
filterFrom.push({ key: ['fromId'], values: ids });
|
||||
}
|
||||
paginateArgs.filters = {
|
||||
mode: FilterMode.Or,
|
||||
filters: [],
|
||||
filterGroups: [
|
||||
{ mode: FilterMode.And, filterGroups: [], filters: filterTo },
|
||||
{ mode: FilterMode.And, filterGroups: [], filters: filterFrom }],
|
||||
};
|
||||
}
|
||||
let relationshipIndices = READ_RELATIONSHIPS_INDICES;
|
||||
if (inferredParameterValues.length > 0) {
|
||||
if (inferredParameterValues.includes('true')) {
|
||||
relationshipIndices = [READ_INDEX_INFERRED_RELATIONSHIPS];
|
||||
} else if (inferredParameterValues.includes('false')) {
|
||||
relationshipIndices = READ_RELATIONSHIPS_INDICES_WITHOUT_INFERRED;
|
||||
};
|
||||
};
|
||||
const relationships = await elList<BasicStoreRelation>(context, user, relationshipIndices, paginateArgs);
|
||||
// compute side ids
|
||||
const addTypeSide = (sideId: string, sideType: string) => {
|
||||
targetValidatedIds.add(sideId);
|
||||
if (sideIdManualInferred.has(sideId)) {
|
||||
const toTypes = sideIdManualInferred.get(sideId);
|
||||
toTypes.add(sideType);
|
||||
sideIdManualInferred.set(sideId, toTypes);
|
||||
} else {
|
||||
const toTypes = new Set();
|
||||
toTypes.add(sideType);
|
||||
sideIdManualInferred.set(sideId, toTypes);
|
||||
}
|
||||
};
|
||||
for (let relIndex = 0; relIndex < relationships.length; relIndex += 1) {
|
||||
await doYield();
|
||||
const relation = relationships[relIndex];
|
||||
const relType = isInferredIndex(relation._index) ? 'inferred' : 'manual';
|
||||
addTypeSide(relation.fromId, relType);
|
||||
addTypeSide(relation.toId, relType);
|
||||
}
|
||||
// We need to ensure elements are filtered according to denormalization rights.
|
||||
const targetValidatedIds = new Set();
|
||||
const sideIdManualInferred = new Map();
|
||||
const { values } = filter;
|
||||
const ids = values.filter((v) => v.key === ID_SUBFILTER).map((f) => f.values).flat();
|
||||
const types = values.filter((v) => v.key === RELATION_TYPE_SUBFILTER).map((f) => f.values).flat();
|
||||
const inferredParameterValues = values.filter((v) => v.key === RELATION_INFERRED_SUBFILTER).map((f) => f.values).flat();
|
||||
const directionForced = R.head(values.filter((v) => v.key === INSTANCE_REGARDING_OF_DIRECTION_FORCED).map((f) => f.values).flat()) ?? false;
|
||||
const directionReverse = R.head(values.filter((v) => v.key === INSTANCE_REGARDING_OF_DIRECTION_REVERSE).map((f) => f.values).flat()) ?? false;
|
||||
// resolve all relationships that target the id values, forcing the type is available
|
||||
const paginateArgs: RepaginateOpts<BasicStoreRelation> = { baseData: true, types };
|
||||
const elementIds = elements.map(({ id }) => id);
|
||||
if (directionForced) {
|
||||
// If a direction is forced, build the filter in the correct direction
|
||||
const directedFilters = [];
|
||||
if (directionReverse) {
|
||||
directedFilters.push({ key: ['fromId'], values: elementIds });
|
||||
if (ids.length > 0) { // Ids can be empty if nothing configured by the user
|
||||
directedFilters.push({ key: ['toId'], values: ids });
|
||||
}
|
||||
return (element: (T & { regardingOfTypes?: string })) => {
|
||||
const accepted = targetValidatedIds.has(element.id);
|
||||
if (accepted) {
|
||||
element.regardingOfTypes = sideIdManualInferred.get(element.id);
|
||||
}
|
||||
return accepted;
|
||||
};
|
||||
} else {
|
||||
directedFilters.push({ key: ['toId'], values: elementIds });
|
||||
if (ids.length > 0) { // Ids can be empty if nothing configured by the user
|
||||
directedFilters.push({ key: ['fromId'], values: ids });
|
||||
}
|
||||
}
|
||||
paginateArgs.filters = { mode: FilterMode.And, filters: directedFilters, filterGroups: [] };
|
||||
} else {
|
||||
// If no direction is setup, create the filter group for both directions
|
||||
const filterTo = [{ key: ['fromId'], values: elementIds }];
|
||||
const filterFrom = [{ key: ['toId'], values: elementIds }];
|
||||
if (ids.length > 0) { // Ids can be empty if nothing configured by the user
|
||||
filterTo.push({ key: ['toId'], values: ids });
|
||||
filterFrom.push({ key: ['fromId'], values: ids });
|
||||
}
|
||||
paginateArgs.filters = {
|
||||
mode: FilterMode.Or,
|
||||
filters: [],
|
||||
filterGroups: [
|
||||
{ mode: FilterMode.And, filterGroups: [], filters: filterTo },
|
||||
{ mode: FilterMode.And, filterGroups: [], filters: filterFrom }],
|
||||
};
|
||||
}
|
||||
let relationshipIndices = READ_RELATIONSHIPS_INDICES;
|
||||
if (inferredParameterValues.length > 0) {
|
||||
if (inferredParameterValues.includes('true')) {
|
||||
relationshipIndices = [READ_INDEX_INFERRED_RELATIONSHIPS];
|
||||
} else if (inferredParameterValues.includes('false')) {
|
||||
relationshipIndices = READ_RELATIONSHIPS_INDICES_WITHOUT_INFERRED;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
const relationships = await elList<BasicStoreRelation>(context, user, relationshipIndices, paginateArgs);
|
||||
// compute side ids
|
||||
const addTypeSide = (sideId: string, sideType: string) => {
|
||||
targetValidatedIds.add(sideId);
|
||||
if (sideIdManualInferred.has(sideId)) {
|
||||
const toTypes = sideIdManualInferred.get(sideId);
|
||||
toTypes.add(sideType);
|
||||
sideIdManualInferred.set(sideId, toTypes);
|
||||
} else {
|
||||
const toTypes = new Set();
|
||||
toTypes.add(sideType);
|
||||
sideIdManualInferred.set(sideId, toTypes);
|
||||
}
|
||||
};
|
||||
for (let relIndex = 0; relIndex < relationships.length; relIndex += 1) {
|
||||
await doYield();
|
||||
const relation = relationships[relIndex];
|
||||
const relType = isInferredIndex(relation._index) ? 'inferred' : 'manual';
|
||||
addTypeSide(relation.fromId, relType);
|
||||
addTypeSide(relation.toId, relType);
|
||||
}
|
||||
return (element: (T & { regardingOfTypes?: string })) => {
|
||||
const accepted = targetValidatedIds.has(element.id);
|
||||
if (accepted) {
|
||||
element.regardingOfTypes = sideIdManualInferred.get(element.id);
|
||||
}
|
||||
return accepted;
|
||||
};
|
||||
};
|
||||
type AttributeValues = {
|
||||
orderMode?: string | null;
|
||||
|
||||
@@ -55,8 +55,8 @@ import { getEntitiesListFromCache } from '../../database/cache';
|
||||
import { ENTITY_TYPE_STATUS } from '../../schema/internalObject';
|
||||
import { IDS_ATTRIBUTES } from '../../domain/attribute-utils';
|
||||
|
||||
export const adaptFilterToRegardingOfFilterKey = async (context: AuthContext, user: AuthUser, filter: Filter) => {
|
||||
const { key: filterKey } = filter;
|
||||
export const adaptFilterToRegardingOfFilterKey = async (context: AuthContext, user: AuthUser, filter: TaggedFilter) => {
|
||||
const { key: filterKey, postFilteringTag } = filter;
|
||||
const regardingFilters = [];
|
||||
const idParameter = filter.values.find((i) => i.key === ID_SUBFILTER);
|
||||
const typeParameter = filter.values.find((i) => i.key === RELATION_TYPE_SUBFILTER);
|
||||
@@ -130,13 +130,13 @@ export const adaptFilterToRegardingOfFilterKey = async (context: AuthContext, us
|
||||
? buildRefRelationKey('*', '*')
|
||||
: types.map((t: string) => buildRefRelationKey(t, '*'));
|
||||
keys.forEach((relKey: string) => {
|
||||
regardingFilters.push({ key: [relKey], operator: filter.operator, values: ['EXISTS'] });
|
||||
regardingFilters.push({ key: [relKey], operator: filter.operator, values: ['EXISTS'], postFilteringTag });
|
||||
});
|
||||
} else {
|
||||
const keys = isEmptyField(types)
|
||||
? buildRefRelationKey('*', '*')
|
||||
: types.flatMap((t: string) => [buildRefRelationKey(t, ID_INTERNAL), buildRefRelationKey(t, ID_INFERRED)]);
|
||||
regardingFilters.push({ key: keys, operator: filter.operator, mode, values: ids });
|
||||
regardingFilters.push({ key: keys, operator: filter.operator, mode, values: ids, postFilteringTag });
|
||||
}
|
||||
return { newFilterGroup: { mode, filters: regardingFilters, filterGroups: [] } };
|
||||
};
|
||||
@@ -683,6 +683,12 @@ const adaptFilterToComputedReliabilityFilterKey = async (context: AuthContext, u
|
||||
|
||||
return { newFilterGroup };
|
||||
};
|
||||
export type TaggedFilter = Filter & { postFilteringTag?: string };
|
||||
export type TaggedFilterGroup = {
|
||||
mode: FilterMode;
|
||||
filters: TaggedFilter[];
|
||||
filterGroups: TaggedFilterGroup[];
|
||||
};
|
||||
/**
|
||||
* Complete the filter if needed for several special filter keys
|
||||
* Some keys need this preprocessing before building the query:
|
||||
@@ -696,11 +702,11 @@ const adaptFilterToComputedReliabilityFilterKey = async (context: AuthContext, u
|
||||
export const completeSpecialFilterKeys = async (
|
||||
context: AuthContext,
|
||||
user: AuthUser,
|
||||
inputFilters: FilterGroup,
|
||||
): Promise<FilterGroup> => {
|
||||
inputFilters: TaggedFilterGroup,
|
||||
): Promise<TaggedFilterGroup> => {
|
||||
const { filters = [], filterGroups = [] } = inputFilters;
|
||||
const finalFilters = [];
|
||||
const finalFilterGroups: FilterGroup[] = [];
|
||||
const finalFilterGroups: TaggedFilterGroup[] = [];
|
||||
for (let index = 0; index < filterGroups.length; index += 1) {
|
||||
const filterGroup = filterGroups[index];
|
||||
const newFilterGroup = await completeSpecialFilterKeys(context, user, filterGroup);
|
||||
|
||||
@@ -2455,9 +2455,9 @@ describe('Complex filters combinations for elastic queries', () => {
|
||||
|
||||
describe('Complex filters regarding of for elastic queries', () => {
|
||||
it('should list entities using basic regarding of filter', async () => {
|
||||
const generateFilters = (withRegardingOf = true, regardingOfOperator = 'eq') => {
|
||||
const generateFilters = (withRegardingOf = true, regardingOfOperator = 'eq', mainMode = 'and') => {
|
||||
return {
|
||||
mode: 'and',
|
||||
mode: mainMode,
|
||||
filters: [
|
||||
{
|
||||
key: 'entity_type',
|
||||
@@ -2494,15 +2494,24 @@ describe('Complex filters regarding of for elastic queries', () => {
|
||||
expect(eqQueryResult.data.globalSearch.edges.length).toEqual(2);
|
||||
expect(eqQueryResult.data.globalSearch.edges[0].node.standard_id).toEqual('intrusion-set--d12c5319-f308-5fef-9336-20484af42084');
|
||||
expect(eqQueryResult.data.globalSearch.edges[1].node.standard_id).toEqual('malware--21c45dbe-54ec-5bb7-b8cd-9f27cc518714');
|
||||
const eqOrQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'eq', 'or') } });
|
||||
expect(eqOrQueryResult.data.globalSearch.edges.length).toEqual(5);
|
||||
expect(eqOrQueryResult.data.globalSearch.edges[0].node.standard_id).toEqual('attack-pattern--a01046cc-192f-5d52-8e75-6e447fae3890');
|
||||
expect(eqOrQueryResult.data.globalSearch.edges[1].node.standard_id).toEqual('attack-pattern--b5c4784e-6ecc-5347-a231-c9739e077dd8');
|
||||
expect(eqOrQueryResult.data.globalSearch.edges[2].node.standard_id).toEqual('intrusion-set--d12c5319-f308-5fef-9336-20484af42084');
|
||||
expect(eqOrQueryResult.data.globalSearch.edges[3].node.standard_id).toEqual('malware--21c45dbe-54ec-5bb7-b8cd-9f27cc518714');
|
||||
expect(eqOrQueryResult.data.globalSearch.edges[4].node.standard_id).toEqual('malware--8a4b5aef-e4a7-524c-92f9-a61c08d1cd85');
|
||||
const notEqQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'not_eq') } });
|
||||
expect(notEqQueryResult.data.globalSearch.edges.length).toEqual(1);
|
||||
expect(notEqQueryResult.data.globalSearch.edges[0].node.standard_id).toEqual('malware--8a4b5aef-e4a7-524c-92f9-a61c08d1cd85');
|
||||
const notEqOrQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'not_eq', 'or') } });
|
||||
expect(notEqOrQueryResult.data.globalSearch.edges.length).toEqual(38);
|
||||
});
|
||||
it('should list entities using complex regarding of filter', async () => {
|
||||
const attackPattern = await storeLoadById(testContext, ADMIN_USER, 'attack-pattern--2fc04aa5-48c1-49ec-919a-b88241ef1d17', ENTITY_TYPE_ATTACK_PATTERN);
|
||||
const generateFilters = (withRegardingOf = true, regardingOfOperator = 'eq') => {
|
||||
const generateFilters = (withRegardingOf = true, regardingOfOperator = 'eq', mainMode = 'and') => {
|
||||
return {
|
||||
mode: 'and',
|
||||
mode: mainMode,
|
||||
filters: [
|
||||
{
|
||||
key: 'entity_type',
|
||||
@@ -2537,16 +2546,20 @@ describe('Complex filters regarding of for elastic queries', () => {
|
||||
const queryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'eq') } });
|
||||
expect(queryResult.data.globalSearch.edges.length).toEqual(1);
|
||||
expect(queryResult.data.globalSearch.edges[0].node.standard_id).toEqual('malware--21c45dbe-54ec-5bb7-b8cd-9f27cc518714');
|
||||
const queryEqOrResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'eq', 'or') } });
|
||||
expect(queryEqOrResult.data.globalSearch.edges.length).toEqual(3);
|
||||
const notEqQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'not_eq') } });
|
||||
expect(notEqQueryResult.data.globalSearch.edges.length).toEqual(2);
|
||||
expect(notEqQueryResult.data.globalSearch.edges[0].node.standard_id).toEqual('intrusion-set--d12c5319-f308-5fef-9336-20484af42084');
|
||||
expect(notEqQueryResult.data.globalSearch.edges[1].node.standard_id).toEqual('malware--8a4b5aef-e4a7-524c-92f9-a61c08d1cd85');
|
||||
const notEqOrQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'not_eq', 'or') } });
|
||||
expect(notEqOrQueryResult.data.globalSearch.edges.length).toEqual(40);
|
||||
});
|
||||
it('should list entities using complex regarding of filter with force direction', async () => {
|
||||
const attackPattern = await storeLoadById(testContext, ADMIN_USER, 'attack-pattern--2fc04aa5-48c1-49ec-919a-b88241ef1d17', ENTITY_TYPE_ATTACK_PATTERN);
|
||||
const generateFilters = (reverse, withRegardingOf = true, regardingOfOperator = 'eq') => {
|
||||
const generateFilters = (reverse, withRegardingOf = true, regardingOfOperator = 'eq', mainMode = 'and') => {
|
||||
return {
|
||||
mode: 'and',
|
||||
mode: mainMode,
|
||||
filters: [
|
||||
{
|
||||
key: 'entity_type',
|
||||
@@ -2583,6 +2596,8 @@ describe('Complex filters regarding of for elastic queries', () => {
|
||||
const queryResultReverse = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, true, 'eq') } });
|
||||
expect(queryResultReverse.data.globalSearch.edges.length).toEqual(1);
|
||||
expect(queryResultReverse.data.globalSearch.edges[0].node.standard_id).toEqual('malware--21c45dbe-54ec-5bb7-b8cd-9f27cc518714');
|
||||
const queryResultReverseOr = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, true, 'eq', 'or') } });
|
||||
expect(queryResultReverseOr.data.globalSearch.edges.length).toEqual(3);
|
||||
const notEqQueryResultReverse = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, true, 'not_eq') } });
|
||||
expect(notEqQueryResultReverse.data.globalSearch.edges.length).toEqual(2);
|
||||
expect(notEqQueryResultReverse.data.globalSearch.edges[0].node.standard_id).toEqual('intrusion-set--d12c5319-f308-5fef-9336-20484af42084');
|
||||
@@ -2590,15 +2605,17 @@ describe('Complex filters regarding of for elastic queries', () => {
|
||||
// reverse = false
|
||||
const queryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(false, true, 'eq') } });
|
||||
expect(queryResult.data.globalSearch.edges.length).toEqual(0); // 0 instead of 1 because of the reverse that implies a post filtering
|
||||
const queryResultOr = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(false, true, 'eq', 'or') } });
|
||||
expect(queryResultOr.data.globalSearch.edges.length).toEqual(3); // 0 instead of 1 because of the reverse that implies a post filtering
|
||||
const notEqQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(false, true, 'not_eq') } });
|
||||
expect(notEqQueryResult.data.globalSearch.edges.length).toEqual(2); // 2 here as post filtering is not applied on not_eq
|
||||
expect(notEqQueryResult.data.globalSearch.edges[0].node.standard_id).toEqual('intrusion-set--d12c5319-f308-5fef-9336-20484af42084');
|
||||
expect(notEqQueryResult.data.globalSearch.edges[1].node.standard_id).toEqual('malware--8a4b5aef-e4a7-524c-92f9-a61c08d1cd85');
|
||||
});
|
||||
it('should list entities using basic regarding of dynamic filter', async () => {
|
||||
const generateFilters = (withRegardingOf = true, regardingOfOperator = 'eq') => {
|
||||
const generateFilters = (withRegardingOf = true, regardingOfOperator = 'eq', mainMode = 'and') => {
|
||||
return {
|
||||
mode: 'and',
|
||||
mode: mainMode,
|
||||
filters: [
|
||||
{
|
||||
key: 'entity_type',
|
||||
@@ -2651,14 +2668,18 @@ describe('Complex filters regarding of for elastic queries', () => {
|
||||
const queryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'eq') } });
|
||||
expect(queryResult.data.globalSearch.edges.length).toEqual(1);
|
||||
expect(queryResult.data.globalSearch.edges[0].node.standard_id).toEqual('malware--21c45dbe-54ec-5bb7-b8cd-9f27cc518714');
|
||||
const queryResultOr = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'eq', 'or') } });
|
||||
expect(queryResultOr.data.globalSearch.edges.length).toEqual(2);
|
||||
const notEqQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'not_eq') } });
|
||||
expect(notEqQueryResult.data.globalSearch.edges.length).toEqual(1);
|
||||
expect(notEqQueryResult.data.globalSearch.edges[0].node.standard_id).toEqual('malware--8a4b5aef-e4a7-524c-92f9-a61c08d1cd85');
|
||||
const notEqOrQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, 'not_eq', 'or') } });
|
||||
expect(notEqOrQueryResult.data.globalSearch.edges.length).toEqual(40);
|
||||
});
|
||||
it('should list entities using multi relationships regarding of filter', async () => {
|
||||
const generateFilters = (withRegardingOf = true, relationships = [], regardingOfOperator = 'eq') => {
|
||||
const generateFilters = (withRegardingOf = true, relationships = [], regardingOfOperator = 'eq', mainMode = 'and') => {
|
||||
return {
|
||||
mode: 'and',
|
||||
mode: mainMode,
|
||||
filters: [
|
||||
{
|
||||
key: 'entity_type',
|
||||
@@ -2699,6 +2720,13 @@ describe('Complex filters regarding of for elastic queries', () => {
|
||||
// - related-to > malware-analysis--8fd6fcd4-81a9-4937-92b8-4e1cbe68f263"
|
||||
let eqQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, ['uses'], 'eq') } });
|
||||
expect(eqQueryResult.data.globalSearch.edges.length).toEqual(1);
|
||||
const eqOrQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, ['uses'], 'eq', 'or') } });
|
||||
expect(eqOrQueryResult.data.globalSearch.edges.length).toEqual(5);
|
||||
expect(eqOrQueryResult.data.globalSearch.edges[0].node.standard_id).toEqual('attack-pattern--a01046cc-192f-5d52-8e75-6e447fae3890');
|
||||
expect(eqOrQueryResult.data.globalSearch.edges[1].node.standard_id).toEqual('attack-pattern--b5c4784e-6ecc-5347-a231-c9739e077dd8');
|
||||
expect(eqOrQueryResult.data.globalSearch.edges[2].node.standard_id).toEqual('intrusion-set--d12c5319-f308-5fef-9336-20484af42084');
|
||||
expect(eqOrQueryResult.data.globalSearch.edges[3].node.standard_id).toEqual('malware--21c45dbe-54ec-5bb7-b8cd-9f27cc518714');
|
||||
expect(eqOrQueryResult.data.globalSearch.edges[4].node.standard_id).toEqual('malware--8a4b5aef-e4a7-524c-92f9-a61c08d1cd85');
|
||||
eqQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, ['uses', 'related-to'], 'eq') } });
|
||||
expect(eqQueryResult.data.globalSearch.edges.length).toEqual(2);
|
||||
expect(eqQueryResult.data.globalSearch.edges[0].node.representative.main).toEqual('Paradise Ransomware');
|
||||
@@ -2712,9 +2740,9 @@ describe('Complex filters regarding of for elastic queries', () => {
|
||||
expect(notEqQueryResult.data.globalSearch.edges[0].node.standard_id).toEqual('malware--8a4b5aef-e4a7-524c-92f9-a61c08d1cd85');
|
||||
});
|
||||
it('should list entities using multi ids regarding of filter', async () => {
|
||||
const generateFilters = (withRegardingOf = true, ids = [], regardingOfOperator = 'eq') => {
|
||||
const generateFilters = (withRegardingOf = true, ids = [], regardingOfOperator = 'eq', mainMode = 'and') => {
|
||||
return {
|
||||
mode: 'and',
|
||||
mode: mainMode,
|
||||
filters: [
|
||||
{
|
||||
key: 'entity_type',
|
||||
@@ -2765,6 +2793,8 @@ describe('Complex filters regarding of for elastic queries', () => {
|
||||
expect(eqQueryResult.data.globalSearch.edges[0].node.standard_id).toEqual('malware--21c45dbe-54ec-5bb7-b8cd-9f27cc518714');
|
||||
expect(eqQueryResult.data.globalSearch.edges[1].node.representative.main).toEqual('Spelevo EK');
|
||||
expect(eqQueryResult.data.globalSearch.edges[1].node.standard_id).toEqual('malware--8a4b5aef-e4a7-524c-92f9-a61c08d1cd85');
|
||||
const eqOrQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, [targetAttack.internal_id, malwareAnalysis.internal_id], 'eq', 'or') } });
|
||||
expect(eqOrQueryResult.data.globalSearch.edges.length).toEqual(4);
|
||||
let notEqQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, [targetAttack.internal_id], 'not_eq') } });
|
||||
expect(notEqQueryResult.data.globalSearch.edges.length).toEqual(1);
|
||||
notEqQueryResult = await queryAsAdmin({ query: LIST_QUERY, variables: { filters: generateFilters(true, [targetAttack.internal_id, malwareAnalysis.internal_id], 'not_eq') } });
|
||||
|
||||
Reference in New Issue
Block a user