From 2cd2ddbf5fcd3f39fe4093022448784fb7b7bdd7 Mon Sep 17 00:00:00 2001 From: Anurag Shivarathri Date: Thu, 15 Jul 2021 18:47:48 +0530 Subject: [PATCH] MM-33580 Granular Data Retention (#5330) * Unable to open previews from search, pinned and mentions * Granular data retention initialised * Added server version check, logic to remove posts & test cases * Restructured reducer & runs the job only one time a day * Updated minimum server version with 5.36 * Reverting client.ts changes * Added client rest calls & removed config check for data retention enabled or not & fixed test case * Added check for license * Update app/actions/views/login.js Co-authored-by: Miguel Alatzar * Update app/store/middlewares/helpers.js Co-authored-by: Miguel Alatzar * Update app/store/middlewares/helpers.js Co-authored-by: Miguel Alatzar * Fixed lint error * Update app/client/rest/general.ts Co-authored-by: Elias Nahum * Update app/client/rest/general.ts Co-authored-by: Elias Nahum * Added redux migration * Updated server version to 5.37 Co-authored-by: Mattermod Co-authored-by: Miguel Alatzar Co-authored-by: Elias Nahum --- app/actions/views/login.js | 4 +- app/actions/views/root.js | 3 +- app/actions/views/user.js | 6 +- app/client/rest/general.ts | 28 ++++- app/mm-redux/actions/general.test.js | 72 ++++++++++++- app/mm-redux/actions/general.ts | 40 ++++++- app/mm-redux/reducers/entities/general.ts | 9 +- app/mm-redux/types/data_retention.ts | 19 ++++ app/mm-redux/types/general.ts | 10 +- app/store/initial_state.js | 5 +- app/store/middlewares/helpers.js | 72 ++++++++++++- app/store/middlewares/middleware.test.js | 126 +++++++++++++++++++++- 12 files changed, 366 insertions(+), 28 deletions(-) create mode 100644 app/mm-redux/types/data_retention.ts diff --git a/app/actions/views/login.js b/app/actions/views/login.js index deea77edb..b4657c99e 100644 --- a/app/actions/views/login.js +++ b/app/actions/views/login.js @@ -23,7 +23,6 @@ export function handleSuccessfulLogin() { await dispatch(loadConfigAndLicense()); const state = getState(); - const config = getConfig(state); const license = getLicense(state); const token = Client4.getToken(); const url = Client4.getUrl(); @@ -46,8 +45,7 @@ export function handleSuccessfulLogin() { }, }); - if (config.DataRetentionEnableMessageDeletion && config.DataRetentionEnableMessageDeletion === 'true' && - license.IsLicensed === 'true' && license.DataRetention === 'true') { + if (license?.IsLicensed === 'true' && license?.DataRetention === 'true') { dispatch(getDataRetentionPolicy()); } else { dispatch({type: GeneralTypes.RECEIVED_DATA_RETENTION_POLICY, data: {}}); diff --git a/app/actions/views/root.js b/app/actions/views/root.js index c84487292..1f47d8198 100644 --- a/app/actions/views/root.js +++ b/app/actions/views/root.js @@ -45,8 +45,7 @@ export function loadConfigAndLicense() { }]; if (currentUserId) { - if (config.DataRetentionEnableMessageDeletion && config.DataRetentionEnableMessageDeletion === 'true' && - license.IsLicensed === 'true' && license.DataRetention === 'true') { + if (license?.IsLicensed === 'true' && license?.DataRetention === 'true') { dispatch(getDataRetentionPolicy()); } else { actions.push({type: GeneralTypes.RECEIVED_DATA_RETENTION_POLICY, data: {}}); diff --git a/app/actions/views/user.js b/app/actions/views/user.js index 9ab0187f6..fd00c231f 100644 --- a/app/actions/views/user.js +++ b/app/actions/views/user.js @@ -11,7 +11,7 @@ import {autoUpdateTimezone} from '@mm-redux/actions/timezone'; import {Client4} from '@client/rest'; import {General} from '@mm-redux/constants'; import EventEmitter from '@mm-redux/utils/event_emitter'; -import {getConfig, getLicense} from '@mm-redux/selectors/entities/general'; +import {getLicense} from '@mm-redux/selectors/entities/general'; import {isTimezoneEnabled} from '@mm-redux/selectors/entities/timezone'; import {getCurrentUserId, getStatusForUserId} from '@mm-redux/selectors/entities/users'; @@ -25,7 +25,6 @@ const HTTP_UNAUTHORIZED = 401; export function completeLogin(user, deviceToken) { return async (dispatch, getState) => { const state = getState(); - const config = getConfig(state); const license = getLicense(state); const token = Client4.getToken(); const url = Client4.getUrl(); @@ -40,8 +39,7 @@ export function completeLogin(user, deviceToken) { } // Data retention - if (config?.DataRetentionEnableMessageDeletion && config?.DataRetentionEnableMessageDeletion === 'true' && - license?.IsLicensed === 'true' && license?.DataRetention === 'true') { + if (license?.IsLicensed === 'true' && license?.DataRetention === 'true') { dispatch(getDataRetentionPolicy()); } else { dispatch({type: GeneralTypes.RECEIVED_DATA_RETENTION_POLICY, data: {}}); diff --git a/app/client/rest/general.ts b/app/client/rest/general.ts index 19e058b1f..4213eaf8d 100644 --- a/app/client/rest/general.ts +++ b/app/client/rest/general.ts @@ -2,10 +2,12 @@ // See LICENSE.txt for license information. import {Config} from '@mm-redux/types/config'; +import {GlobalDataRetentionPolicy, ChannelDataRetentionPolicy, TeamDataRetentionPolicy} from '@mm-redux/types/data_retention'; import {Role} from '@mm-redux/types/roles'; import {Dictionary} from '@mm-redux/types/utilities'; import {buildQueryString} from '@mm-redux/utils/helpers'; +import {PER_PAGE_DEFAULT} from './constants'; import ClientError from './error'; export interface ClientGeneralMix { @@ -15,7 +17,15 @@ export interface ClientGeneralMix { getClientConfigOld: () => Promise; getClientLicenseOld: () => Promise; getTimezones: () => Promise; - getDataRetentionPolicy: () => Promise; + getGlobalDataRetentionPolicy: () => Promise; + getTeamDataRetentionPolicies: (userId: string, page?: number, perPage?: number) => Promise<{ + policies: TeamDataRetentionPolicy[]; + total_count: number; + }>; + getChannelDataRetentionPolicies: (userId: string, page?: number, perPage?: number) => Promise<{ + policies: ChannelDataRetentionPolicy[]; + total_count: number; + }>; getRolesByNames: (rolesNames: string[]) => Promise; getRedirectLocation: (urlParam: string) => Promise>; } @@ -72,13 +82,27 @@ const ClientGeneral = (superclass: any) => class extends superclass { ); }; - getDataRetentionPolicy = () => { + getGlobalDataRetentionPolicy = () => { return this.doFetch( `${this.getDataRetentionRoute()}/policy`, {method: 'get'}, ); }; + getTeamDataRetentionPolicies = (userId: string, page = 0, perPage = PER_PAGE_DEFAULT) => { + return this.doFetch( + `${this.getBaseRoute()}/users/${userId}/data_retention/team_policies${buildQueryString({page, per_page: perPage})}`, + {method: 'get'}, + ); + }; + + getChannelDataRetentionPolicies = (userId: string, page = 0, perPage = PER_PAGE_DEFAULT) => { + return this.doFetch( + `${this.getBaseRoute()}/users/${userId}/data_retention/channel_policies${buildQueryString({page, per_page: perPage})}`, + {method: 'get'}, + ); + }; + getRolesByNames = async (rolesNames: string[]) => { return this.doFetch( `${this.getRolesRoute()}/names`, diff --git a/app/mm-redux/actions/general.test.js b/app/mm-redux/actions/general.test.js index 56d2b0188..c5be5b769 100644 --- a/app/mm-redux/actions/general.test.js +++ b/app/mm-redux/actions/general.test.js @@ -8,6 +8,7 @@ import {FormattedError} from './helpers.ts'; import {GeneralTypes} from '@mm-redux/action_types'; import * as Actions from '@mm-redux/actions/general'; import {Client4} from '@client/rest'; +import {PER_PAGE_DEFAULT} from '@client/rest/constants'; import TestHelper from 'test/test_helper'; import configureStore from 'test/test_store'; @@ -100,22 +101,85 @@ describe('Actions.General', () => { }); it('getDataRetentionPolicy', async () => { - const responseData = { + const globalPolicyResponse = { message_deletion_enabled: true, file_deletion_enabled: false, message_retention_cutoff: Date.now(), file_retention_cutoff: 0, }; + const channelPoliciesResponse1 = { + policies: [{ + post_duration: 5, + channel_id: 'channe1', + }], + total_count: 2, + }; + + const channelPoliciesResponse2 = { + policies: [{ + post_duration: 2, + channel_id: 'channe2', + }], + total_count: 2, + }; + + const teamPoliciesResponse1 = { + policies: [{ + post_duration: 1, + team_id: 'team1', + }], + total_count: 2, + }; + + const teamPoliciesResponse2 = { + policies: [{ + post_duration: 2, + team_id: 'team2', + }], + total_count: 2, + }; + + const userId = ''; + nock(Client4.getBaseRoute()). get('/data_retention/policy'). query(true). - reply(200, responseData); + reply(200, globalPolicyResponse). + get(`/users/${userId}/data_retention/channel_policies`). + query({ + page: 0, + per_page: PER_PAGE_DEFAULT, + }). + reply(200, channelPoliciesResponse1). + get(`/users/${userId}/data_retention/channel_policies`). + query({ + page: 1, + per_page: PER_PAGE_DEFAULT, + }). + reply(200, channelPoliciesResponse2). + get(`/users/${userId}/data_retention/team_policies`). + query({ + page: 0, + per_page: PER_PAGE_DEFAULT, + }). + reply(200, teamPoliciesResponse1). + get(`/users/${userId}/data_retention/team_policies`). + query({ + page: 1, + per_page: PER_PAGE_DEFAULT, + }). + reply(200, teamPoliciesResponse2); + await store.dispatch({type: GeneralTypes.RECEIVED_SERVER_VERSION, data: '5.37.0'}); await Actions.getDataRetentionPolicy()(store.dispatch, store.getState); await TestHelper.wait(100); - const {dataRetentionPolicy} = store.getState().entities.general; - assert.deepEqual(dataRetentionPolicy, responseData); + const {dataRetention} = store.getState().entities.general; + assert.deepEqual(dataRetention.policies, { + global: globalPolicyResponse, + channels: [...channelPoliciesResponse1.policies, ...channelPoliciesResponse2.policies], + teams: [...teamPoliciesResponse1.policies, ...teamPoliciesResponse2.policies], + }); }); it('getTimezones', async () => { diff --git a/app/mm-redux/actions/general.ts b/app/mm-redux/actions/general.ts index 46bc3b25b..082e39139 100644 --- a/app/mm-redux/actions/general.ts +++ b/app/mm-redux/actions/general.ts @@ -4,8 +4,10 @@ import {Client4} from '@client/rest'; import {GeneralTypes} from '@mm-redux/action_types'; +import {getCurrentUserId} from '@mm-redux/selectors/entities/common'; import {getServerVersion} from '@mm-redux/selectors/entities/general'; import {isMinimumServerVersion} from '@mm-redux/utils/helpers'; +import {TeamDataRetentionPolicy, ChannelDataRetentionPolicy} from '@mm-redux/types/data_retention'; import {GeneralState} from '@mm-redux/types/general'; import {logLevel} from '@mm-redux/types/client4'; import {GetStateFunc, DispatchFunc, ActionFunc, batchActions} from '@mm-redux/types/actions'; @@ -72,7 +74,27 @@ export function getDataRetentionPolicy(): ActionFunc { return async (dispatch: DispatchFunc, getState: GetStateFunc) => { let data; try { - data = await Client4.getDataRetentionPolicy(); + const state = getState(); + const userId = getCurrentUserId(state); + const globalPolicy = await Client4.getGlobalDataRetentionPolicy(); + + let teamPolicies: TeamDataRetentionPolicy[] = []; + let channelPolicies: ChannelDataRetentionPolicy[] = []; + + if (isMinimumServerVersion(getServerVersion(getState()), 5, 37)) { + teamPolicies = await getAllGranularDataRetentionPolicies({ + userId, + }); + channelPolicies = await getAllGranularDataRetentionPolicies({ + isChannel: true, + userId, + }); + } + data = { + global: globalPolicy, + teams: teamPolicies, + channels: channelPolicies, + }; } catch (error) { forceLogoutIfNecessary(error, dispatch, getState); dispatch(batchActions([ @@ -93,6 +115,22 @@ export function getDataRetentionPolicy(): ActionFunc { }; } +async function getAllGranularDataRetentionPolicies(options: { + isChannel?: boolean; + page?: number; + policies?: TeamDataRetentionPolicy[] | ChannelDataRetentionPolicy[]; + userId: string; +}): Promise { + const {isChannel, page = 0, policies = [], userId} = options; + const api = isChannel ? 'getChannelDataRetentionPolicies' : 'getTeamDataRetentionPolicies'; + const data = await Client4[api](userId, page); + policies.push(...data.policies); + if (policies.length < data.total_count) { + await getAllGranularDataRetentionPolicies({...options, policies, page: page + 1}); + } + return policies; +} + export function getLicenseConfig(): ActionFunc { return bindClientFunc({ clientFunc: Client4.getClientLicenseOld, diff --git a/app/mm-redux/reducers/entities/general.ts b/app/mm-redux/reducers/entities/general.ts index 6c3876a23..fda52f9bc 100644 --- a/app/mm-redux/reducers/entities/general.ts +++ b/app/mm-redux/reducers/entities/general.ts @@ -43,10 +43,13 @@ function credentials(state: any = {}, action: GenericAction) { } } -function dataRetentionPolicy(state: any = {}, action: GenericAction) { +function dataRetention(state: any = {}, action: GenericAction) { switch (action.type) { case GeneralTypes.RECEIVED_DATA_RETENTION_POLICY: - return action.data; + return { + ...state, + policies: action.data, + }; default: return state; } @@ -95,7 +98,7 @@ export default combineReducers({ appState, credentials, config, - dataRetentionPolicy, + dataRetention, deviceToken, license, serverVersion, diff --git a/app/mm-redux/types/data_retention.ts b/app/mm-redux/types/data_retention.ts new file mode 100644 index 000000000..c2414dd4b --- /dev/null +++ b/app/mm-redux/types/data_retention.ts @@ -0,0 +1,19 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +export type GlobalDataRetentionPolicy = { + file_deletion_enabled: boolean; + file_retention_cutoff: number; + message_deletion_enabled: boolean; + message_retention_cutoff: number; +} + +export type TeamDataRetentionPolicy = { + post_duration: number; + team_id?: string; +} + +export type ChannelDataRetentionPolicy = { + post_duration: number; + channel_id?: string; +} diff --git a/app/mm-redux/types/general.ts b/app/mm-redux/types/general.ts index c2554141c..9b468da5e 100644 --- a/app/mm-redux/types/general.ts +++ b/app/mm-redux/types/general.ts @@ -2,12 +2,20 @@ // See LICENSE.txt for license information. import {Config} from './config'; +import {ChannelDataRetentionPolicy, GlobalDataRetentionPolicy, TeamDataRetentionPolicy} from './data_retention'; export type GeneralState = { appState: boolean; credentials: any; config: Partial; - dataRetentionPolicy: any; + dataRetention: { + policies: { + channels: ChannelDataRetentionPolicy[]; + global: GlobalDataRetentionPolicy; + teams: TeamDataRetentionPolicy[]; + }; + lastCleanUpAt: Date; + }; deviceToken: string; license: any; serverVersion: string; diff --git a/app/store/initial_state.js b/app/store/initial_state.js index 90ba4748b..5f131627b 100644 --- a/app/store/initial_state.js +++ b/app/store/initial_state.js @@ -13,7 +13,10 @@ const state = { appState: false, credentials: {}, config: {}, - dataRetentionPolicy: {}, + dataRetention: { + policies: {}, + lastCleanUpAt: null, + }, deviceToken: '', license: {}, serverVersion: '', diff --git a/app/store/middlewares/helpers.js b/app/store/middlewares/helpers.js index 7d3962474..bfc2df71e 100644 --- a/app/store/middlewares/helpers.js +++ b/app/store/middlewares/helpers.js @@ -39,17 +39,64 @@ export function cleanUpState(payload, keepCurrent = false) { files: {}, fileIdsByPostId: {}, }, + general: { + ...payload.entities?.general, + dataRetention: { + ...payload.entities?.general?.dataRetention, + }, + }, }; - let retentionPeriod = 0; - if (payload.entities?.general?.dataRetentionPolicy?.message_deletion_enabled) { - retentionPeriod = payload.entities.general.dataRetentionPolicy.message_retention_cutoff; + // Migrate old data retention policy value to the granular structure + if (nextEntities.general.dataRetentionPolicy) { + nextEntities.general.dataRetention = { + ...nextEntities.general.dataRetention, + policies: { + ...nextEntities.general.dataRetention?.policies?.global, + global: nextEntities.general.dataRetentionPolicy, + }, + }; + delete nextEntities.general.dataRetentionPolicy; + } + + let globalRetentionCutoff = 0; + const channelsRetentionCutoff = {}; + + const {policies, lastCleanUpAt} = nextEntities.general.dataRetention || {}; + + const lastCleanedToday = new Date(lastCleanUpAt).toDateString() === new Date().toDateString(); + if (policies && (!lastCleanUpAt || !lastCleanedToday)) { + if (policies.global?.message_deletion_enabled) { + globalRetentionCutoff = policies.global.message_retention_cutoff; + } + + // Channels from team policies + policies?.teams?.forEach((policy) => { + const channels = payload.entities?.channels?.channelsInTeam?.[policy.team_id]; + if (channels) { + const cutoff = getRetentionCutoffFromPolicy(policy); + channels.forEach((channel_id) => { + channelsRetentionCutoff[channel_id] = cutoff; + }); + } + }); + + // Add/Override from channels only policies + policies?.channels?.forEach((policy) => { + channelsRetentionCutoff[policy.channel_id] = getRetentionCutoffFromPolicy(policy); + }); + + // Update the last cleanup date only when we have atleast one policy + if (globalRetentionCutoff || Object.keys(channelsRetentionCutoff).length) { + nextEntities.general.dataRetention.lastCleanUpAt = new Date(); + } } const postIdsToKeep = []; // Keep the last 60 posts in each recently viewed channel nextEntities.posts.postsInChannel = cleanUpPostsInChannel(payload.entities.posts?.postsInChannel, lastChannelForTeam, keepCurrent ? currentChannelId : ''); + postIdsToKeep.push(...getAllFromPostsInChannel(nextEntities.posts.postsInChannel)); // Keep any posts that appear in search results @@ -85,7 +132,7 @@ export function cleanUpState(payload, keepCurrent = false) { const post = payload.entities.posts.posts[postId]; if (post) { - if (retentionPeriod && post.create_at < retentionPeriod) { + if (shouldRemovePost(globalRetentionCutoff, channelsRetentionCutoff, post)) { // This post has been removed by data retention, so don't keep it removeFromPostsInChannel(nextEntities.posts.postsInChannel, post.channel_id, postId); @@ -225,6 +272,23 @@ export function getAllFromPostsInChannel(postsInChannel) { return postIds; } +// Returns cutoff time for the policy +function getRetentionCutoffFromPolicy(policy) { + const periodDate = new Date(); + periodDate.setDate(periodDate.getDate() - policy.post_duration); + return periodDate.getTime(); +} + +function shouldRemovePost(globalRetentionCutoff, channelsRetentionCutoff, post) { + // If cutoff is found for the channel + if (channelsRetentionCutoff[post.channel_id]) { + return post.create_at < channelsRetentionCutoff[post.channel_id]; + } else if (globalRetentionCutoff) { + return post.create_at < globalRetentionCutoff; + } + return false; +} + function removeFromPostsInChannel(postsInChannel, channelId, postId) { const postsForChannel = postsInChannel[channelId]; diff --git a/app/store/middlewares/middleware.test.js b/app/store/middlewares/middleware.test.js index 59eb68b0e..ee14058bb 100644 --- a/app/store/middlewares/middleware.test.js +++ b/app/store/middlewares/middleware.test.js @@ -189,9 +189,13 @@ describe('cleanUpState', () => { fileIdsByPostId: {}, }, general: { - dataRetentionPolicy: { - message_deletion_enabled: true, - message_retention_cutoff: 1000, + dataRetention: { + policies: { + global: { + message_deletion_enabled: true, + message_retention_cutoff: 1000, + }, + }, }, }, posts: { @@ -230,6 +234,122 @@ describe('cleanUpState', () => { expect(result.entities.search.flagged).toEqual(['post1', 'post2', 'post3']); }); + test('should migrate dataRetentionPolicy key to granular data retention structure', () => { + const dataRetentionPolicy = { + file_deletion_enabled: false, + file_retention_cutoff: 0, + message_deletion_enabled: false, + message_retention_cutoff: 0, + }; + + const state = { + entities: { + general: { + dataRetentionPolicy, + }, + }, + }; + + const result = cleanUpState(state); + + expect(result.entities.general.dataRetentionPolicy).toBeUndefined(); + expect(result.entities.general.dataRetention.policies.global).toEqual(dataRetentionPolicy); + }); + + test('should remove post because of granular data retention', () => { + const getCreateAtBeforeDays = (days) => { + const date = new Date(); + date.setDate(date.getDate() - days); + return date.getTime(); + }; + const state = { + entities: { + channels: { + currentChannelId: 'channel1', + channelsInTeam: { + team1: new Set(['team1_channel1', 'team1_channel2']), + team2: new Set(['team2_channel1', 'team2_channel2']), + }, + }, + files: { + fileIdsByPostId: {}, + }, + general: { + dataRetention: { + policies: { + teams: [{ + post_duration: 5, + team_id: 'team1', + }, { + post_duration: 10, + team_id: 'team2', + }], + channels: [{ + post_duration: 2, + channel_id: 'team1_channel1', + }], + }, + }, + }, + posts: { + posts: { + + // Team 1 - Channel 1, Channel Policy - 2 Days + post1: {id: 'post1', channel_id: 'team1_channel1', create_at: getCreateAtBeforeDays(1)}, + post2: {id: 'post2', channel_id: 'team1_channel1', create_at: getCreateAtBeforeDays(3)}, // X + + // Team 1 - Channel 2, Team Policy - 5 Days + post3: {id: 'post3', channel_id: 'team1_channel2', create_at: getCreateAtBeforeDays(3)}, + post4: {id: 'post4', channel_id: 'team1_channel2', create_at: getCreateAtBeforeDays(6)}, // X + + // Team 2, Channel 1 & 2, Team Policy - 10 Days + post5: {id: 'post5', channel_id: 'team2_channel1', create_at: getCreateAtBeforeDays(9)}, + post6: {id: 'post6', channel_id: 'team2_channel2', create_at: getCreateAtBeforeDays(11)}, // X + }, + postsInChannel: { + team1_channel1: [ + {order: ['post1', 'post2'], recent: true}, + ], + team1_channel2: [ + {order: ['post3', 'post4'], recent: true}, + ], + team2_channel1: [ + {order: ['post5'], recent: true}, + ], + team2_channel2: [ + {order: ['post6'], recent: true}, + ], + }, + postsInThread: {}, + reactions: {}, + }, + search: { + results: [], + flagged: [], + }, + }, + views: { + team: { + lastChannelForTeam: { + team1: ['team1_channel1', 'team1_channel2'], + team2: ['team2_channel1', 'team2_channel2'], + }, + }, + }, + }; + + const result = cleanUpState(state); + + expect(result.entities.posts.posts.post1).toBeDefined(); + expect(result.entities.posts.posts.post2).toBeUndefined(); + + expect(result.entities.posts.posts.post3).toBeDefined(); + expect(result.entities.posts.posts.post4).toBeUndefined(); + + expect(result.entities.posts.posts.post5).toBeDefined(); + expect(result.entities.posts.posts.post6).toBeUndefined(); + }); + test('should keep failed pending post', () => { const state = { entities: {