diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts
index a53a6e00810b..38bc84ae9af3 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts
@@ -965,6 +965,41 @@ describe('state_helpers', () => {
         );
       });
 
+      it('should not carry over label when operation and field change at the same time', () => {
+        expect(
+          replaceColumn({
+            layer: {
+              indexPatternId: '1',
+              columnOrder: ['col1'],
+              columns: {
+                col1: {
+                  label: 'My custom label',
+                  customLabel: true,
+                  dataType: 'date',
+                  isBucketed: true,
+
+                  // Private
+                  operationType: 'date_histogram',
+                  sourceField: 'timestamp',
+                  params: {
+                    interval: 'h',
+                  },
+                },
+              },
+            },
+            indexPattern,
+            columnId: 'col1',
+            op: 'terms',
+            field: indexPattern.fields[4],
+            visualizationGroups: [],
+          }).columns.col1
+        ).toEqual(
+          expect.objectContaining({
+            label: 'Top values of source',
+          })
+        );
+      });
+
       it('should carry over label on operation switch when customLabel flag on previousColumn is set', () => {
         expect(
           replaceColumn({
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts
index b42cdbd24e65..56fbb8edef5b 100644
--- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts
+++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts
@@ -838,12 +838,14 @@ function applyReferenceTransition({
   );
 }
 
-function copyCustomLabel(
-  newColumn: IndexPatternColumn,
-  previousOptions: { customLabel?: boolean; label: string }
-) {
+function copyCustomLabel(newColumn: IndexPatternColumn, previousOptions: IndexPatternColumn) {
   const adjustedColumn = { ...newColumn };
-  if (previousOptions.customLabel) {
+  const operationChanged = newColumn.operationType !== previousOptions.operationType;
+  const fieldChanged =
+    ('sourceField' in newColumn && newColumn.sourceField) !==
+    ('sourceField' in previousOptions && previousOptions.sourceField);
+  // only copy custom label if either used operation or used field stayed the same
+  if (previousOptions.customLabel && (!operationChanged || !fieldChanged)) {
     adjustedColumn.customLabel = true;
     adjustedColumn.label = previousOptions.label;
   }