MM-33580 Granular Data Retention (#5330)

* Unable to open previews from search, pinned and mentions

* Granular data retention initialised

* Added server version check, logic to remove posts & test cases

* Restructured reducer & runs the job only one time a day

* Updated minimum server version with 5.36

* Reverting client.ts changes

* Added client rest calls & removed config check for data retention enabled or not & fixed test case

* Added check for license

* Update app/actions/views/login.js

Co-authored-by: Miguel Alatzar <migbot@users.noreply.github.com>

* Update app/store/middlewares/helpers.js

Co-authored-by: Miguel Alatzar <migbot@users.noreply.github.com>

* Update app/store/middlewares/helpers.js

Co-authored-by: Miguel Alatzar <migbot@users.noreply.github.com>

* Fixed lint error

* Update app/client/rest/general.ts

Co-authored-by: Elias Nahum <nahumhbl@gmail.com>

* Update app/client/rest/general.ts

Co-authored-by: Elias Nahum <nahumhbl@gmail.com>

* Added redux migration

* Updated server version to 5.37

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Miguel Alatzar <migbot@users.noreply.github.com>
Co-authored-by: Elias Nahum <nahumhbl@gmail.com>

app/actions/views/login.js

@@ -23,7 +23,6 @@ export function handleSuccessfulLogin() {
         await dispatch(loadConfigAndLicense());
 
         const state = getState();
-        const config = getConfig(state);
         const license = getLicense(state);
         const token = Client4.getToken();
         const url = Client4.getUrl();
@@ -46,8 +45,7 @@ export function handleSuccessfulLogin() {
             },
         });
 
-        if (config.DataRetentionEnableMessageDeletion && config.DataRetentionEnableMessageDeletion === 'true' &&
-            license.IsLicensed === 'true' && license.DataRetention === 'true') {
+        if (license?.IsLicensed === 'true' && license?.DataRetention === 'true') {
             dispatch(getDataRetentionPolicy());
         } else {
             dispatch({type: GeneralTypes.RECEIVED_DATA_RETENTION_POLICY, data: {}});

app/actions/views/root.js

@@ -45,8 +45,7 @@ export function loadConfigAndLicense() {
             }];
 
             if (currentUserId) {
-                if (config.DataRetentionEnableMessageDeletion && config.DataRetentionEnableMessageDeletion === 'true' &&
-                    license.IsLicensed === 'true' && license.DataRetention === 'true') {
+                if (license?.IsLicensed === 'true' && license?.DataRetention === 'true') {
                     dispatch(getDataRetentionPolicy());
                 } else {
                     actions.push({type: GeneralTypes.RECEIVED_DATA_RETENTION_POLICY, data: {}});

app/actions/views/user.js

@@ -11,7 +11,7 @@ import {autoUpdateTimezone} from '@mm-redux/actions/timezone';
 import {Client4} from '@client/rest';
 import {General} from '@mm-redux/constants';
 import EventEmitter from '@mm-redux/utils/event_emitter';
-import {getConfig, getLicense} from '@mm-redux/selectors/entities/general';
+import {getLicense} from '@mm-redux/selectors/entities/general';
 import {isTimezoneEnabled} from '@mm-redux/selectors/entities/timezone';
 import {getCurrentUserId, getStatusForUserId} from '@mm-redux/selectors/entities/users';
 
@@ -25,7 +25,6 @@ const HTTP_UNAUTHORIZED = 401;
 export function completeLogin(user, deviceToken) {
     return async (dispatch, getState) => {
         const state = getState();
-        const config = getConfig(state);
         const license = getLicense(state);
         const token = Client4.getToken();
         const url = Client4.getUrl();
@@ -40,8 +39,7 @@ export function completeLogin(user, deviceToken) {
         }
 
         // Data retention
-        if (config?.DataRetentionEnableMessageDeletion && config?.DataRetentionEnableMessageDeletion === 'true' &&
-            license?.IsLicensed === 'true' && license?.DataRetention === 'true') {
+        if (license?.IsLicensed === 'true' && license?.DataRetention === 'true') {
             dispatch(getDataRetentionPolicy());
         } else {
             dispatch({type: GeneralTypes.RECEIVED_DATA_RETENTION_POLICY, data: {}});

app/client/rest/general.ts

@@ -2,10 +2,12 @@
 // See LICENSE.txt for license information.
 
 import {Config} from '@mm-redux/types/config';
+import {GlobalDataRetentionPolicy, ChannelDataRetentionPolicy, TeamDataRetentionPolicy} from '@mm-redux/types/data_retention';
 import {Role} from '@mm-redux/types/roles';
 import {Dictionary} from '@mm-redux/types/utilities';
 import {buildQueryString} from '@mm-redux/utils/helpers';
 
+import {PER_PAGE_DEFAULT} from './constants';
 import ClientError from './error';
 
 export interface ClientGeneralMix {
@@ -15,7 +17,15 @@ export interface ClientGeneralMix {
     getClientConfigOld: () => Promise<Config>;
     getClientLicenseOld: () => Promise<any>;
     getTimezones: () => Promise<string[]>;
-    getDataRetentionPolicy: () => Promise<any>;
+    getGlobalDataRetentionPolicy: () => Promise<GlobalDataRetentionPolicy[]>;
+    getTeamDataRetentionPolicies: (userId: string, page?: number, perPage?: number) => Promise<{
+        policies: TeamDataRetentionPolicy[];
+        total_count: number;
+    }>;
+    getChannelDataRetentionPolicies: (userId: string, page?: number, perPage?: number) => Promise<{
+        policies: ChannelDataRetentionPolicy[];
+        total_count: number;
+    }>;
     getRolesByNames: (rolesNames: string[]) => Promise<Role[]>;
     getRedirectLocation: (urlParam: string) => Promise<Dictionary<string>>;
 }
@@ -72,13 +82,27 @@ const ClientGeneral = (superclass: any) => class extends superclass {
         );
     };
 
-    getDataRetentionPolicy = () => {
+    getGlobalDataRetentionPolicy = () => {
         return this.doFetch(
             `${this.getDataRetentionRoute()}/policy`,
             {method: 'get'},
         );
     };
 
+    getTeamDataRetentionPolicies = (userId: string, page = 0, perPage = PER_PAGE_DEFAULT) => {
+        return this.doFetch(
+            `${this.getBaseRoute()}/users/${userId}/data_retention/team_policies${buildQueryString({page, per_page: perPage})}`,
+            {method: 'get'},
+        );
+    };
+
+    getChannelDataRetentionPolicies = (userId: string, page = 0, perPage = PER_PAGE_DEFAULT) => {
+        return this.doFetch(
+            `${this.getBaseRoute()}/users/${userId}/data_retention/channel_policies${buildQueryString({page, per_page: perPage})}`,
+            {method: 'get'},
+        );
+    };
+
     getRolesByNames = async (rolesNames: string[]) => {
         return this.doFetch(
             `${this.getRolesRoute()}/names`,

app/mm-redux/actions/general.test.js

@@ -8,6 +8,7 @@ import {FormattedError} from './helpers.ts';
 import {GeneralTypes} from '@mm-redux/action_types';
 import * as Actions from '@mm-redux/actions/general';
 import {Client4} from '@client/rest';
+import {PER_PAGE_DEFAULT} from '@client/rest/constants';
 
 import TestHelper from 'test/test_helper';
 import configureStore from 'test/test_store';
@@ -100,22 +101,85 @@ describe('Actions.General', () => {
     });
 
     it('getDataRetentionPolicy', async () => {
-        const responseData = {
+        const globalPolicyResponse = {
             message_deletion_enabled: true,
             file_deletion_enabled: false,
             message_retention_cutoff: Date.now(),
             file_retention_cutoff: 0,
         };
 
+        const channelPoliciesResponse1 = {
+            policies: [{
+                post_duration: 5,
+                channel_id: 'channe1',
+            }],
+            total_count: 2,
+        };
+
+        const channelPoliciesResponse2 = {
+            policies: [{
+                post_duration: 2,
+                channel_id: 'channe2',
+            }],
+            total_count: 2,
+        };
+
+        const teamPoliciesResponse1 = {
+            policies: [{
+                post_duration: 1,
+                team_id: 'team1',
+            }],
+            total_count: 2,
+        };
+
+        const teamPoliciesResponse2 = {
+            policies: [{
+                post_duration: 2,
+                team_id: 'team2',
+            }],
+            total_count: 2,
+        };
+
+        const userId = '';
+
         nock(Client4.getBaseRoute()).
             get('/data_retention/policy').
             query(true).
-            reply(200, responseData);
+            reply(200, globalPolicyResponse).
+            get(`/users/${userId}/data_retention/channel_policies`).
+            query({
+                page: 0,
+                per_page: PER_PAGE_DEFAULT,
+            }).
+            reply(200, channelPoliciesResponse1).
+            get(`/users/${userId}/data_retention/channel_policies`).
+            query({
+                page: 1,
+                per_page: PER_PAGE_DEFAULT,
+            }).
+            reply(200, channelPoliciesResponse2).
+            get(`/users/${userId}/data_retention/team_policies`).
+            query({
+                page: 0,
+                per_page: PER_PAGE_DEFAULT,
+            }).
+            reply(200, teamPoliciesResponse1).
+            get(`/users/${userId}/data_retention/team_policies`).
+            query({
+                page: 1,
+                per_page: PER_PAGE_DEFAULT,
+            }).
+            reply(200, teamPoliciesResponse2);
 
+        await store.dispatch({type: GeneralTypes.RECEIVED_SERVER_VERSION, data: '5.37.0'});
         await Actions.getDataRetentionPolicy()(store.dispatch, store.getState);
         await TestHelper.wait(100);
-        const {dataRetentionPolicy} = store.getState().entities.general;
-        assert.deepEqual(dataRetentionPolicy, responseData);
+        const {dataRetention} = store.getState().entities.general;
+        assert.deepEqual(dataRetention.policies, {
+            global: globalPolicyResponse,
+            channels: [...channelPoliciesResponse1.policies, ...channelPoliciesResponse2.policies],
+            teams: [...teamPoliciesResponse1.policies, ...teamPoliciesResponse2.policies],
+        });
     });
 
     it('getTimezones', async () => {

app/mm-redux/actions/general.ts

@@ -4,8 +4,10 @@ import {Client4} from '@client/rest';
 
 import {GeneralTypes} from '@mm-redux/action_types';
 
+import {getCurrentUserId} from '@mm-redux/selectors/entities/common';
 import {getServerVersion} from '@mm-redux/selectors/entities/general';
 import {isMinimumServerVersion} from '@mm-redux/utils/helpers';
+import {TeamDataRetentionPolicy, ChannelDataRetentionPolicy} from '@mm-redux/types/data_retention';
 import {GeneralState} from '@mm-redux/types/general';
 import {logLevel} from '@mm-redux/types/client4';
 import {GetStateFunc, DispatchFunc, ActionFunc, batchActions} from '@mm-redux/types/actions';
@@ -72,7 +74,27 @@ export function getDataRetentionPolicy(): ActionFunc {
     return async (dispatch: DispatchFunc, getState: GetStateFunc) => {
         let data;
         try {
-            data = await Client4.getDataRetentionPolicy();
+            const state = getState();
+            const userId = getCurrentUserId(state);
+            const globalPolicy = await Client4.getGlobalDataRetentionPolicy();
+
+            let teamPolicies: TeamDataRetentionPolicy[] = [];
+            let channelPolicies: ChannelDataRetentionPolicy[] = [];
+
+            if (isMinimumServerVersion(getServerVersion(getState()), 5, 37)) {
+                teamPolicies = await getAllGranularDataRetentionPolicies({
+                    userId,
+                });
+                channelPolicies = await getAllGranularDataRetentionPolicies({
+                    isChannel: true,
+                    userId,
+                });
+            }
+            data = {
+                global: globalPolicy,
+                teams: teamPolicies,
+                channels: channelPolicies,
+            };
         } catch (error) {
             forceLogoutIfNecessary(error, dispatch, getState);
             dispatch(batchActions([
@@ -93,6 +115,22 @@ export function getDataRetentionPolicy(): ActionFunc {
     };
 }
 
+async function getAllGranularDataRetentionPolicies(options: {
+    isChannel?: boolean;
+    page?: number;
+    policies?: TeamDataRetentionPolicy[] | ChannelDataRetentionPolicy[];
+    userId: string;
+}): Promise<TeamDataRetentionPolicy[] | ChannelDataRetentionPolicy[]> {
+    const {isChannel, page = 0, policies = [], userId} = options;
+    const api = isChannel ? 'getChannelDataRetentionPolicies' : 'getTeamDataRetentionPolicies';
+    const data = await Client4[api](userId, page);
+    policies.push(...data.policies);
+    if (policies.length < data.total_count) {
+        await getAllGranularDataRetentionPolicies({...options, policies, page: page + 1});
+    }
+    return policies;
+}
+
 export function getLicenseConfig(): ActionFunc {
     return bindClientFunc({
         clientFunc: Client4.getClientLicenseOld,

app/mm-redux/reducers/entities/general.ts

@@ -43,10 +43,13 @@ function credentials(state: any = {}, action: GenericAction) {
     }
 }
 
-function dataRetentionPolicy(state: any = {}, action: GenericAction) {
+function dataRetention(state: any = {}, action: GenericAction) {
     switch (action.type) {
     case GeneralTypes.RECEIVED_DATA_RETENTION_POLICY:
-        return action.data;
+        return {
+            ...state,
+            policies: action.data,
+        };
     default:
         return state;
     }
@@ -95,7 +98,7 @@ export default combineReducers({
     appState,
     credentials,
     config,
-    dataRetentionPolicy,
+    dataRetention,
     deviceToken,
     license,
     serverVersion,

app/mm-redux/types/data_retention.ts

@@ -0,0 +1,19 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+export type GlobalDataRetentionPolicy = {
+    file_deletion_enabled: boolean;
+    file_retention_cutoff: number;
+    message_deletion_enabled: boolean;
+    message_retention_cutoff: number;
+}
+
+export type TeamDataRetentionPolicy = {
+    post_duration: number;
+    team_id?: string;
+}
+
+export type ChannelDataRetentionPolicy = {
+    post_duration: number;
+    channel_id?: string;
+}

app/mm-redux/types/general.ts

@@ -2,12 +2,20 @@
 // See LICENSE.txt for license information.
 
 import {Config} from './config';
+import {ChannelDataRetentionPolicy, GlobalDataRetentionPolicy, TeamDataRetentionPolicy} from './data_retention';
 
 export type GeneralState = {
     appState: boolean;
     credentials: any;
     config: Partial<Config>;
-    dataRetentionPolicy: any;
+    dataRetention: {
+        policies: {
+            channels: ChannelDataRetentionPolicy[];
+            global: GlobalDataRetentionPolicy;
+            teams: TeamDataRetentionPolicy[];
+        };
+        lastCleanUpAt: Date;
+    };
     deviceToken: string;
     license: any;
     serverVersion: string;

app/store/initial_state.js

@@ -13,7 +13,10 @@ const state = {
             appState: false,
             credentials: {},
             config: {},
-            dataRetentionPolicy: {},
+            dataRetention: {
+                policies: {},
+                lastCleanUpAt: null,
+            },
             deviceToken: '',
             license: {},
             serverVersion: '',

app/store/middlewares/helpers.js

@@ -39,17 +39,64 @@ export function cleanUpState(payload, keepCurrent = false) {
             files: {},
             fileIdsByPostId: {},
         },
+        general: {
+            ...payload.entities?.general,
+            dataRetention: {
+                ...payload.entities?.general?.dataRetention,
+            },
+        },
     };
 
-    let retentionPeriod = 0;
-    if (payload.entities?.general?.dataRetentionPolicy?.message_deletion_enabled) {
-        retentionPeriod = payload.entities.general.dataRetentionPolicy.message_retention_cutoff;
+    // Migrate old data retention policy value to the granular structure
+    if (nextEntities.general.dataRetentionPolicy) {
+        nextEntities.general.dataRetention = {
+            ...nextEntities.general.dataRetention,
+            policies: {
+                ...nextEntities.general.dataRetention?.policies?.global,
+                global: nextEntities.general.dataRetentionPolicy,
+            },
+        };
+        delete nextEntities.general.dataRetentionPolicy;
+    }
+
+    let globalRetentionCutoff = 0;
+    const channelsRetentionCutoff = {};
+
+    const {policies, lastCleanUpAt} = nextEntities.general.dataRetention || {};
+
+    const lastCleanedToday = new Date(lastCleanUpAt).toDateString() === new Date().toDateString();
+    if (policies && (!lastCleanUpAt || !lastCleanedToday)) {
+        if (policies.global?.message_deletion_enabled) {
+            globalRetentionCutoff = policies.global.message_retention_cutoff;
+        }
+
+        // Channels from team policies
+        policies?.teams?.forEach((policy) => {
+            const channels = payload.entities?.channels?.channelsInTeam?.[policy.team_id];
+            if (channels) {
+                const cutoff = getRetentionCutoffFromPolicy(policy);
+                channels.forEach((channel_id) => {
+                    channelsRetentionCutoff[channel_id] = cutoff;
+                });
+            }
+        });
+
+        // Add/Override from channels only policies
+        policies?.channels?.forEach((policy) => {
+            channelsRetentionCutoff[policy.channel_id] = getRetentionCutoffFromPolicy(policy);
+        });
+
+        // Update the last cleanup date only when we have atleast one policy
+        if (globalRetentionCutoff || Object.keys(channelsRetentionCutoff).length) {
+            nextEntities.general.dataRetention.lastCleanUpAt = new Date();
+        }
     }
 
     const postIdsToKeep = [];
 
     // Keep the last 60 posts in each recently viewed channel
     nextEntities.posts.postsInChannel = cleanUpPostsInChannel(payload.entities.posts?.postsInChannel, lastChannelForTeam, keepCurrent ? currentChannelId : '');
+
     postIdsToKeep.push(...getAllFromPostsInChannel(nextEntities.posts.postsInChannel));
 
     // Keep any posts that appear in search results
@@ -85,7 +132,7 @@ export function cleanUpState(payload, keepCurrent = false) {
             const post = payload.entities.posts.posts[postId];
 
             if (post) {
-                if (retentionPeriod && post.create_at < retentionPeriod) {
+                if (shouldRemovePost(globalRetentionCutoff, channelsRetentionCutoff, post)) {
                     // This post has been removed by data retention, so don't keep it
                     removeFromPostsInChannel(nextEntities.posts.postsInChannel, post.channel_id, postId);
 
@@ -225,6 +272,23 @@ export function getAllFromPostsInChannel(postsInChannel) {
     return postIds;
 }
 
+// Returns cutoff time for the policy
+function getRetentionCutoffFromPolicy(policy) {
+    const periodDate = new Date();
+    periodDate.setDate(periodDate.getDate() - policy.post_duration);
+    return periodDate.getTime();
+}
+
+function shouldRemovePost(globalRetentionCutoff, channelsRetentionCutoff, post) {
+    // If cutoff is found for the channel
+    if (channelsRetentionCutoff[post.channel_id]) {
+        return post.create_at < channelsRetentionCutoff[post.channel_id];
+    } else if (globalRetentionCutoff) {
+        return post.create_at < globalRetentionCutoff;
+    }
+    return false;
+}
+
 function removeFromPostsInChannel(postsInChannel, channelId, postId) {
     const postsForChannel = postsInChannel[channelId];
 

app/store/middlewares/middleware.test.js

@@ -189,9 +189,13 @@ describe('cleanUpState', () => {
                     fileIdsByPostId: {},
                 },
                 general: {
-                    dataRetentionPolicy: {
-                        message_deletion_enabled: true,
-                        message_retention_cutoff: 1000,
+                    dataRetention: {
+                        policies: {
+                            global: {
+                                message_deletion_enabled: true,
+                                message_retention_cutoff: 1000,
+                            },
+                        },
                     },
                 },
                 posts: {
@@ -230,6 +234,122 @@ describe('cleanUpState', () => {
         expect(result.entities.search.flagged).toEqual(['post1', 'post2', 'post3']);
     });
 
+    test('should migrate dataRetentionPolicy key to granular data retention structure', () => {
+        const dataRetentionPolicy = {
+            file_deletion_enabled: false,
+            file_retention_cutoff: 0,
+            message_deletion_enabled: false,
+            message_retention_cutoff: 0,
+        };
+
+        const state = {
+            entities: {
+                general: {
+                    dataRetentionPolicy,
+                },
+            },
+        };
+
+        const result = cleanUpState(state);
+
+        expect(result.entities.general.dataRetentionPolicy).toBeUndefined();
+        expect(result.entities.general.dataRetention.policies.global).toEqual(dataRetentionPolicy);
+    });
+
+    test('should remove post because of granular data retention', () => {
+        const getCreateAtBeforeDays = (days) => {
+            const date = new Date();
+            date.setDate(date.getDate() - days);
+            return date.getTime();
+        };
+        const state = {
+            entities: {
+                channels: {
+                    currentChannelId: 'channel1',
+                    channelsInTeam: {
+                        team1: new Set(['team1_channel1', 'team1_channel2']),
+                        team2: new Set(['team2_channel1', 'team2_channel2']),
+                    },
+                },
+                files: {
+                    fileIdsByPostId: {},
+                },
+                general: {
+                    dataRetention: {
+                        policies: {
+                            teams: [{
+                                post_duration: 5,
+                                team_id: 'team1',
+                            }, {
+                                post_duration: 10,
+                                team_id: 'team2',
+                            }],
+                            channels: [{
+                                post_duration: 2,
+                                channel_id: 'team1_channel1',
+                            }],
+                        },
+                    },
+                },
+                posts: {
+                    posts: {
+
+                        // Team 1 - Channel 1, Channel Policy - 2 Days
+                        post1: {id: 'post1', channel_id: 'team1_channel1', create_at: getCreateAtBeforeDays(1)},
+                        post2: {id: 'post2', channel_id: 'team1_channel1', create_at: getCreateAtBeforeDays(3)}, // X
+
+                        // Team 1 - Channel 2, Team Policy - 5 Days
+                        post3: {id: 'post3', channel_id: 'team1_channel2', create_at: getCreateAtBeforeDays(3)},
+                        post4: {id: 'post4', channel_id: 'team1_channel2', create_at: getCreateAtBeforeDays(6)}, // X
+
+                        // Team 2, Channel 1 & 2, Team Policy - 10 Days
+                        post5: {id: 'post5', channel_id: 'team2_channel1', create_at: getCreateAtBeforeDays(9)},
+                        post6: {id: 'post6', channel_id: 'team2_channel2', create_at: getCreateAtBeforeDays(11)}, // X
+                    },
+                    postsInChannel: {
+                        team1_channel1: [
+                            {order: ['post1', 'post2'], recent: true},
+                        ],
+                        team1_channel2: [
+                            {order: ['post3', 'post4'], recent: true},
+                        ],
+                        team2_channel1: [
+                            {order: ['post5'], recent: true},
+                        ],
+                        team2_channel2: [
+                            {order: ['post6'], recent: true},
+                        ],
+                    },
+                    postsInThread: {},
+                    reactions: {},
+                },
+                search: {
+                    results: [],
+                    flagged: [],
+                },
+            },
+            views: {
+                team: {
+                    lastChannelForTeam: {
+                        team1: ['team1_channel1', 'team1_channel2'],
+                        team2: ['team2_channel1', 'team2_channel2'],
+                    },
+                },
+            },
+        };
+
+        const result = cleanUpState(state);
+
+        expect(result.entities.posts.posts.post1).toBeDefined();
+        expect(result.entities.posts.posts.post2).toBeUndefined();
+
+        expect(result.entities.posts.posts.post3).toBeDefined();
+        expect(result.entities.posts.posts.post4).toBeUndefined();
+
+        expect(result.entities.posts.posts.post5).toBeDefined();
+        expect(result.entities.posts.posts.post6).toBeUndefined();
+    });
+
     test('should keep failed pending post', () => {
         const state = {
             entities: {