From 7c5e9f3e335a2630f72d7766e53618514def0a8b Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 10 Jul 2025 10:19:04 -0400 Subject: [PATCH 01/19] rebase master --- doc/api/graphql/reference/_index.md | 2 + ee/app/finders/ee/projects_finder.rb | 12 ++++ .../graphql/ee/resolvers/projects_resolver.rb | 7 +- .../ai/active_context/code/repository.rb | 4 ++ ee/app/models/ee/project.rb | 8 +++ ...ode_embeddings_indexed_projects_filter.yml | 10 +++ ee/spec/finders/ee/projects_finder_spec.rb | 72 +++++++++++++++++++ .../ee/resolvers/projects_resolver_spec.rb | 32 +++++++++ .../ai/active_context/code/repository_spec.rb | 48 +++++++++++++ ee/spec/models/ee/project_spec.rb | 40 +++++++++++ .../import_export/project/import_export.yml | 1 + spec/lib/gitlab/import_export/all_models.yml | 1 + 12 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 ee/config/feature_flags/experiment/allow_with_code_embeddings_indexed_projects_filter.yml diff --git a/doc/api/graphql/reference/_index.md b/doc/api/graphql/reference/_index.md index 462f91fff6ed29..823b15c3fc95fb 100644 --- a/doc/api/graphql/reference/_index.md +++ b/doc/api/graphql/reference/_index.md @@ -1331,6 +1331,7 @@ four standard [pagination arguments](#pagination-arguments): | `topics` | [`[String!]`](#string) | Filter projects by topics. | | `trending` | [`Boolean`](#boolean) | Return only projects that are trending. | | `visibilityLevel` | [`VisibilityLevelsEnum`](#visibilitylevelsenum) | Filter projects by visibility level. | +| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. | | `withIssuesEnabled` | [`Boolean`](#boolean) | Return only projects with issues enabled. | | `withMergeRequestsEnabled` | [`Boolean`](#boolean) | Return only projects with merge requests enabled. | @@ -35637,6 +35638,7 @@ four standard [pagination arguments](#pagination-arguments): | `topics` | [`[String!]`](#string) | Filter projects by topics. | | `trending` | [`Boolean`](#boolean) | Return only projects that are trending. | | `visibilityLevel` | [`VisibilityLevelsEnum`](#visibilitylevelsenum) | Filter projects by visibility level. | +| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. | | `withIssuesEnabled` | [`Boolean`](#boolean) | Return only projects with issues enabled. | | `withMergeRequestsEnabled` | [`Boolean`](#boolean) | Return only projects with merge requests enabled. | diff --git a/ee/app/finders/ee/projects_finder.rb b/ee/app/finders/ee/projects_finder.rb index ee37ce1dbbe40d..c953fa941abfe8 100644 --- a/ee/app/finders/ee/projects_finder.rb +++ b/ee/app/finders/ee/projects_finder.rb @@ -14,6 +14,8 @@ module EE # filter_expired_saml_session_projects: boolean # active: boolean - Whether to include projects that are neither archived or marked # for deletion. + # with_code_embeddings_indexed: boolean - Whether to include projects that have + # indexed embeddings for their code module ProjectsFinder include Gitlab::Auth::Saml::SsoSessionFilterable extend ::Gitlab::Utils::Override @@ -26,6 +28,7 @@ def filter_projects(collection) collection = by_plans(collection) collection = by_feature_available(collection) collection = by_hidden(collection) + collection = by_code_embeddings_indexed(collection) by_saml_sso_session(collection) end @@ -45,6 +48,15 @@ def by_feature_available(collection) end end + def by_code_embeddings_indexed(items) + if ::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) && + ::Feature.enabled?(:allow_with_code_embeddings_indexed_projects_filter, current_user) + items.with_ready_active_context_code_repository + else + items + end + end + def by_hidden(items) params[:include_hidden].present? ? items : items.not_hidden end diff --git a/ee/app/graphql/ee/resolvers/projects_resolver.rb b/ee/app/graphql/ee/resolvers/projects_resolver.rb index 75560a5d4d6b87..9d769c35507d4d 100644 --- a/ee/app/graphql/ee/resolvers/projects_resolver.rb +++ b/ee/app/graphql/ee/resolvers/projects_resolver.rb @@ -11,6 +11,11 @@ module ProjectsResolver required: false, description: 'Include hidden projects.' + argument :with_code_embeddings_indexed, GraphQL::Types::Boolean, + required: false, + experiment: { milestone: '18.2' }, + description: 'Include projects with indexed code embeddings.' + before_connection_authorization do |projects, current_user| ::Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects, current_user).execute ::Preloaders::UserMemberRolesInProjectsPreloader.new(projects: projects, user: current_user).execute @@ -22,7 +27,7 @@ module ProjectsResolver override :finder_params def finder_params(args) super(args) - .merge(args.slice(:include_hidden)) + .merge(args.slice(:include_hidden, :with_code_embeddings_indexed)) .merge(filter_expired_saml_session_projects: true) end end diff --git a/ee/app/models/ai/active_context/code/repository.rb b/ee/app/models/ai/active_context/code/repository.rb index c4e78586cf54ee..c10071298d31af 100644 --- a/ee/app/models/ai/active_context/code/repository.rb +++ b/ee/app/models/ai/active_context/code/repository.rb @@ -54,6 +54,10 @@ class Repository < ApplicationRecord joins(:active_context_connection).where(active_context_connection: { active: true }) } + scope :ready_with_active_connection, -> { + by_state(:ready).with_active_connection + } + private def set_last_commit diff --git a/ee/app/models/ee/project.rb b/ee/app/models/ee/project.rb index 2c719db8d2bd7b..a5da204ac3a9a8 100644 --- a/ee/app/models/ee/project.rb +++ b/ee/app/models/ee/project.rb @@ -232,6 +232,10 @@ def lock_for_confirmation!(id) has_many :configured_ai_catalog_items, class_name: '::Ai::Catalog::ItemConsumer', inverse_of: :project + has_one :ready_active_context_code_repository, + -> { ready_with_active_connection }, + class_name: 'Ai::ActiveContext::Code::Repository' + elastic_index_dependant_association :issues, on_change: :visibility_level elastic_index_dependant_association :issues, on_change: :archived elastic_index_dependant_association :work_items, on_change: :visibility_level @@ -445,6 +449,10 @@ def lock_for_confirmation!(id) "AND zoekt_repositories.zoekt_index_id = ?)", index_id) } + scope :with_ready_active_context_code_repository, -> { + joins(:ready_active_context_code_repository) + } + delegate :shared_runners_seconds, to: :statistics, allow_nil: true delegate :ci_minutes_usage, to: :shared_runners_limit_namespace diff --git a/ee/config/feature_flags/experiment/allow_with_code_embeddings_indexed_projects_filter.yml b/ee/config/feature_flags/experiment/allow_with_code_embeddings_indexed_projects_filter.yml new file mode 100644 index 00000000000000..9ab027d074c290 --- /dev/null +++ b/ee/config/feature_flags/experiment/allow_with_code_embeddings_indexed_projects_filter.yml @@ -0,0 +1,10 @@ +--- +name: allow_with_code_embeddings_indexed_projects_filter +description: +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/547112 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/195699 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/553819 +milestone: '18.2' +group: group::code creation +type: experiment +default_enabled: false diff --git a/ee/spec/finders/ee/projects_finder_spec.rb b/ee/spec/finders/ee/projects_finder_spec.rb index d1d86e7d02d420..90b047ec6bdc9a 100644 --- a/ee/spec/finders/ee/projects_finder_spec.rb +++ b/ee/spec/finders/ee/projects_finder_spec.rb @@ -108,6 +108,78 @@ end end + context 'filter by code_embeddings_indexed' do + let_it_be(:params) { { with_code_embeddings_indexed: true } } + + let_it_be(:namespace) { create(:group) } + let!(:code_embeddings_enabled_namespace) do + create(:ai_active_context_code_enabled_namespace, namespace: namespace) + end + + let!(:code_embeddings_repository_1) do + create( + :ai_active_context_code_repository, + project: ultimate_project, + enabled_namespace: code_embeddings_enabled_namespace + ) + end + + let!(:code_embeddings_repository_2) do + create( + :ai_active_context_code_repository, + project: ultimate_project2, + enabled_namespace: code_embeddings_enabled_namespace + ) + end + + let!(:code_embeddings_repository_3) do + create( + :ai_active_context_code_repository, + project: premium_project, + enabled_namespace: code_embeddings_enabled_namespace + ) + end + + context 'when ai_active_context_connection is inactive' do + it 'returns no project' do + is_expected.to be_empty + end + end + + context 'when ai_active_context_connection is active' do + before do + code_embeddings_enabled_namespace.active_context_connection.reload.update!(active: true) + end + + context 'when code_embeddings_repository is not ready' do + it 'returns no project code_embeddings_repository' do + is_expected.to be_empty + end + end + + context 'when code_embeddings_repository are ready' do + before do + code_embeddings_repository_1.update!(state: :ready) + code_embeddings_repository_3.update!(state: :ready) + end + + it 'returns project with ready code_embeddings_repository' do + is_expected.to contain_exactly(ultimate_project, premium_project) + end + + context 'when ff allow_with_code_embeddings_indexed_projects_filter is false' do + before do + stub_feature_flags(allow_with_code_embeddings_indexed_projects_filter: false) + end + + it 'returns no project code_embeddings_repository' do + is_expected.to contain_exactly(ultimate_project, ultimate_project2, premium_project, no_plan_project) + end + end + end + end + end + private def create_project(plan, visibility = :public) diff --git a/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb b/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb index 6ee03fe1d3f30f..fc47681f4f5303 100644 --- a/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb +++ b/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb @@ -36,5 +36,37 @@ it { is_expected.to contain_exactly(project, project_marked_for_deletion) } end + + context 'when requesting with_code_embeddings_indexed' do + let!(:filters) { { with_code_embeddings_indexed: true } } + + let_it_be(:namespace) { create(:group) } + let_it_be(:code_embeddings_enabled_namespace) do + create(:ai_active_context_code_enabled_namespace, namespace: namespace) + end + + let!(:code_embeddings_repository) do + create( + :ai_active_context_code_repository, + project: project, + enabled_namespace: code_embeddings_enabled_namespace + ) + end + + it 'returns no projects' do + is_expected.to eq [] + end + + context 'when there are active code_embeddings_repository' do + before do + code_embeddings_repository.active_context_connection.update!(active: true) + code_embeddings_repository.update!(state: :ready) + end + + it 'returns projects with active code_embeddings_repository' do + is_expected.to contain_exactly(project) + end + end + end end end diff --git a/ee/spec/models/ai/active_context/code/repository_spec.rb b/ee/spec/models/ai/active_context/code/repository_spec.rb index 2bdd7e6a1cfc4e..207bbb694f1fb5 100644 --- a/ee/spec/models/ai/active_context/code/repository_spec.rb +++ b/ee/spec/models/ai/active_context/code/repository_spec.rb @@ -157,6 +157,54 @@ expect(result).to contain_exactly(repository_with_active_connection) end end + + describe '.pending_with_active_connection' do + let_it_be(:active_connection) { create(:ai_active_context_connection) } + let_it_be(:inactive_connection) { create(:ai_active_context_connection, :inactive) } + let_it_be(:pending_with_active) do + create(:ai_active_context_code_repository, state: :pending, active_context_connection: active_connection) + end + + let_it_be(:ready_with_active) do + create(:ai_active_context_code_repository, state: :ready, active_context_connection: active_connection) + end + + let_it_be(:pending_with_inactive) do + create(:ai_active_context_code_repository, state: :pending, active_context_connection: inactive_connection) + end + + it 'returns pending repositories with active connections' do + result = described_class.pending_with_active_connection + + expect(result).to contain_exactly(pending_with_active) + end + end + + describe '.ready_with_active_connection' do + it 'does not return repositories with inactive connection' do + expect(described_class.ready_with_active_connection).to be_empty + end + + context 'when connection is active' do + before do + repository.active_context_connection.update!(active: true) + end + + it 'returns repositories with active connection' do + expect(described_class.ready_with_active_connection).to be_empty + end + + context 'when connection is active and repository is ready' do + before do + repository.update!(state: :ready) + end + + it 'returns repositories with active connection' do + expect(described_class.ready_with_active_connection).to contain_exactly(repository) + end + end + end + end end describe 'table partitioning' do diff --git a/ee/spec/models/ee/project_spec.rb b/ee/spec/models/ee/project_spec.rb index fabadca96af40d..a8bf1f0cb35dd4 100644 --- a/ee/spec/models/ee/project_spec.rb +++ b/ee/spec/models/ee/project_spec.rb @@ -855,6 +855,46 @@ expect(described_class.without_security_setting).to match_array([project_without_security_setting]) end end + + describe '.with_ready_active_context_code_repositories' do + let_it_be(:project) { create(:project) } + let_it_be(:namespace) { create(:group) } + let_it_be(:code_embeddings_enabled_namespace) do + create(:ai_active_context_code_enabled_namespace, namespace: namespace) + end + + let(:code_embeddings_repository) do + create( + :ai_active_context_code_repository, + project: project, + enabled_namespace: code_embeddings_enabled_namespace + ) + end + + it 'return no project' do + expect(described_class.with_ready_active_context_code_repository).to be_empty + end + + context "when embedding repository is ready" do + before do + code_embeddings_repository.update!(state: :ready) + end + + it 'returns projects ai_active_context_code_repositories ready' do + expect(described_class.with_ready_active_context_code_repository).to be_empty + end + + context "when embedding repository is ready" do + before do + code_embeddings_repository.active_context_connection.update!(active: true) + end + + it 'returns projects ai_active_context_code_repositories ready' do + expect(described_class.with_ready_active_context_code_repository).to contain_exactly(project) + end + end + end + end end describe 'validations' do diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 1e4ab2a91b1e4d..e5a0bf6cd11ade 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -1374,6 +1374,7 @@ ee: excluded_attributes: project: - :vulnerability_hooks_integrations + - :ready_active_context_code_repository approval_rules: - :created_at - :updated_at diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index f010aeaa5e485f..bdc4c4aed44ca5 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -618,6 +618,7 @@ project: - alert_hooks_integrations - incident_hooks_integrations - vulnerability_hooks_integrations +- ready_active_context_code_repository - apple_app_store_integration - google_cloud_platform_artifact_registry_integration - google_cloud_platform_workload_identity_federation_integration -- GitLab From beeb2fe2b3c3001b6b33637728f22c90771d0963 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 7 Jul 2025 16:28:35 -0400 Subject: [PATCH 02/19] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Pam Artiaga --- ee/app/graphql/ee/resolvers/projects_resolver.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ee/app/graphql/ee/resolvers/projects_resolver.rb b/ee/app/graphql/ee/resolvers/projects_resolver.rb index 9d769c35507d4d..f455b10b667e44 100644 --- a/ee/app/graphql/ee/resolvers/projects_resolver.rb +++ b/ee/app/graphql/ee/resolvers/projects_resolver.rb @@ -14,7 +14,8 @@ module ProjectsResolver argument :with_code_embeddings_indexed, GraphQL::Types::Boolean, required: false, experiment: { milestone: '18.2' }, - description: 'Include projects with indexed code embeddings.' + description: "Include projects with indexed code embeddings. " \ + "This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`." before_connection_authorization do |projects, current_user| ::Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects, current_user).execute -- GitLab From b43ac42592a5029f458819a867e88058c93926b5 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Mon, 7 Jul 2025 16:41:22 -0400 Subject: [PATCH 03/19] update the graphql document --- doc/api/graphql/reference/_index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/api/graphql/reference/_index.md b/doc/api/graphql/reference/_index.md index 823b15c3fc95fb..9941525b20e88f 100644 --- a/doc/api/graphql/reference/_index.md +++ b/doc/api/graphql/reference/_index.md @@ -1331,7 +1331,7 @@ four standard [pagination arguments](#pagination-arguments): | `topics` | [`[String!]`](#string) | Filter projects by topics. | | `trending` | [`Boolean`](#boolean) | Return only projects that are trending. | | `visibilityLevel` | [`VisibilityLevelsEnum`](#visibilitylevelsenum) | Filter projects by visibility level. | -| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. | +| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`. | | `withIssuesEnabled` | [`Boolean`](#boolean) | Return only projects with issues enabled. | | `withMergeRequestsEnabled` | [`Boolean`](#boolean) | Return only projects with merge requests enabled. | @@ -35638,7 +35638,7 @@ four standard [pagination arguments](#pagination-arguments): | `topics` | [`[String!]`](#string) | Filter projects by topics. | | `trending` | [`Boolean`](#boolean) | Return only projects that are trending. | | `visibilityLevel` | [`VisibilityLevelsEnum`](#visibilitylevelsenum) | Filter projects by visibility level. | -| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. | +| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`. | | `withIssuesEnabled` | [`Boolean`](#boolean) | Return only projects with issues enabled. | | `withMergeRequestsEnabled` | [`Boolean`](#boolean) | Return only projects with merge requests enabled. | -- GitLab From b31bbd74c2936f28ed77c2af839c0c51a98e1838 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Mon, 7 Jul 2025 16:44:04 -0400 Subject: [PATCH 04/19] update the scope ready_with_active_connection to use ready enum --- ee/app/models/ai/active_context/code/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/app/models/ai/active_context/code/repository.rb b/ee/app/models/ai/active_context/code/repository.rb index c10071298d31af..483211d11faf15 100644 --- a/ee/app/models/ai/active_context/code/repository.rb +++ b/ee/app/models/ai/active_context/code/repository.rb @@ -55,7 +55,7 @@ class Repository < ApplicationRecord } scope :ready_with_active_connection, -> { - by_state(:ready).with_active_connection + ready.with_active_connection } private -- GitLab From 648b4c51c4f724042c56806c25e6b0eddfaacb60 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Mon, 7 Jul 2025 16:44:24 -0400 Subject: [PATCH 05/19] update the scope ready_with_active_connection to use ready enum --- ee/app/models/ai/active_context/code/repository.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ee/app/models/ai/active_context/code/repository.rb b/ee/app/models/ai/active_context/code/repository.rb index 483211d11faf15..55ce59f0bf0c7c 100644 --- a/ee/app/models/ai/active_context/code/repository.rb +++ b/ee/app/models/ai/active_context/code/repository.rb @@ -54,9 +54,7 @@ class Repository < ApplicationRecord joins(:active_context_connection).where(active_context_connection: { active: true }) } - scope :ready_with_active_connection, -> { - ready.with_active_connection - } + scope :ready_with_active_connection, -> { ready.with_active_connection } private -- GitLab From cdb25ab8cccb189fd4691c2449bdd31397176b2d Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 10 Jul 2025 10:20:57 -0400 Subject: [PATCH 06/19] rebase master --- ee/app/models/ai/active_context/code/repository.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/ee/app/models/ai/active_context/code/repository.rb b/ee/app/models/ai/active_context/code/repository.rb index 55ce59f0bf0c7c..ac581e70ce5047 100644 --- a/ee/app/models/ai/active_context/code/repository.rb +++ b/ee/app/models/ai/active_context/code/repository.rb @@ -54,6 +54,7 @@ class Repository < ApplicationRecord joins(:active_context_connection).where(active_context_connection: { active: true }) } + scope :pending_with_active_connection, -> { by_state(:pending).with_active_connection } scope :ready_with_active_connection, -> { ready.with_active_connection } private -- GitLab From 399d9506b8f6920b8abc96f3fb8f03ddde08dadd Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 10 Jul 2025 11:30:21 -0400 Subject: [PATCH 07/19] rebase master --- app/graphql/resolvers/projects_resolver.rb | 1 - ee/app/finders/ee/projects_finder.rb | 6 +++++- ee/app/models/ee/project.rb | 7 +++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/graphql/resolvers/projects_resolver.rb b/app/graphql/resolvers/projects_resolver.rb index a27990b3940521..e46c2ebf23b5b4 100644 --- a/app/graphql/resolvers/projects_resolver.rb +++ b/app/graphql/resolvers/projects_resolver.rb @@ -69,7 +69,6 @@ def resolve_with_lookahead(**args) projects = ProjectsFinder .new(current_user: current_user, params: finder_params(args), project_ids_relation: parse_gids(args[:ids])) .execute - apply_lookahead(projects) end diff --git a/ee/app/finders/ee/projects_finder.rb b/ee/app/finders/ee/projects_finder.rb index c953fa941abfe8..7e313d0c7b94a8 100644 --- a/ee/app/finders/ee/projects_finder.rb +++ b/ee/app/finders/ee/projects_finder.rb @@ -49,9 +49,13 @@ def by_feature_available(collection) end def by_code_embeddings_indexed(items) + # project ids is required because active_context_code_repositories is a partitioned table + # project_id is used to do the partition prune + return items unless project_ids_relation.present? + if ::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) && ::Feature.enabled?(:allow_with_code_embeddings_indexed_projects_filter, current_user) - items.with_ready_active_context_code_repository + items.with_ready_active_context_code_repository_project_ids project_ids_relation else items end diff --git a/ee/app/models/ee/project.rb b/ee/app/models/ee/project.rb index a5da204ac3a9a8..57e7b23c4863ca 100644 --- a/ee/app/models/ee/project.rb +++ b/ee/app/models/ee/project.rb @@ -453,6 +453,13 @@ def lock_for_confirmation!(id) joins(:ready_active_context_code_repository) } + # can't filter with id on projects because project_id is used as partitioning key + # on p_ai_active_context_code_repositories, query by project_id enables partition pruning + scope :with_ready_active_context_code_repository_project_ids, ->(project_ids) { + joins(:ready_active_context_code_repository) + .where(p_ai_active_context_code_repositories: { project_id: project_ids }) + } + delegate :shared_runners_seconds, to: :statistics, allow_nil: true delegate :ci_minutes_usage, to: :shared_runners_limit_namespace -- GitLab From 6c3974d104aa35c04eb6fb77b6760c68fcc191e2 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 10 Jul 2025 14:54:16 -0400 Subject: [PATCH 08/19] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Terri Chu --- ee/app/graphql/ee/resolvers/projects_resolver.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/app/graphql/ee/resolvers/projects_resolver.rb b/ee/app/graphql/ee/resolvers/projects_resolver.rb index f455b10b667e44..183ae126dadda2 100644 --- a/ee/app/graphql/ee/resolvers/projects_resolver.rb +++ b/ee/app/graphql/ee/resolvers/projects_resolver.rb @@ -15,7 +15,7 @@ module ProjectsResolver required: false, experiment: { milestone: '18.2' }, description: "Include projects with indexed code embeddings. " \ - "This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`." + "Requires `ids` to be sent. This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`." before_connection_authorization do |projects, current_user| ::Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects, current_user).execute -- GitLab From 5352eec8a708a9420ca06bab3d721850e661366d Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 10 Jul 2025 14:56:06 -0400 Subject: [PATCH 09/19] rebase master --- app/graphql/resolvers/projects_resolver.rb | 1 + ee/app/graphql/ee/resolvers/projects_resolver.rb | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/graphql/resolvers/projects_resolver.rb b/app/graphql/resolvers/projects_resolver.rb index e46c2ebf23b5b4..a27990b3940521 100644 --- a/app/graphql/resolvers/projects_resolver.rb +++ b/app/graphql/resolvers/projects_resolver.rb @@ -69,6 +69,7 @@ def resolve_with_lookahead(**args) projects = ProjectsFinder .new(current_user: current_user, params: finder_params(args), project_ids_relation: parse_gids(args[:ids])) .execute + apply_lookahead(projects) end diff --git a/ee/app/graphql/ee/resolvers/projects_resolver.rb b/ee/app/graphql/ee/resolvers/projects_resolver.rb index 183ae126dadda2..6f5afc5476674c 100644 --- a/ee/app/graphql/ee/resolvers/projects_resolver.rb +++ b/ee/app/graphql/ee/resolvers/projects_resolver.rb @@ -15,7 +15,8 @@ module ProjectsResolver required: false, experiment: { milestone: '18.2' }, description: "Include projects with indexed code embeddings. " \ - "Requires `ids` to be sent. This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`." + "Requires `ids` to be sent. This is only applied with the FF " \ + "`allow_with_code_embeddings_indexed_projects_filter=true`." before_connection_authorization do |projects, current_user| ::Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects, current_user).execute -- GitLab From b04a2b0b26a77664feadad8cf7258066ec421efe Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 10 Jul 2025 16:10:51 -0400 Subject: [PATCH 10/19] update the unit tests --- doc/api/graphql/reference/_index.md | 4 ++-- .../graphql/ee/resolvers/projects_resolver.rb | 7 ++++++ .../ai/active_context/code/repository.rb | 1 - ee/app/models/ee/project.rb | 9 ++++---- ee/spec/finders/ee/projects_finder_spec.rb | 4 ++++ .../ee/resolvers/projects_resolver_spec.rb | 6 ++++- .../ai/active_context/code/repository_spec.rb | 22 ------------------- ee/spec/models/ee/project_spec.rb | 20 +++++++++++++---- 8 files changed, 39 insertions(+), 34 deletions(-) diff --git a/doc/api/graphql/reference/_index.md b/doc/api/graphql/reference/_index.md index 9941525b20e88f..0a8b555ab21c6c 100644 --- a/doc/api/graphql/reference/_index.md +++ b/doc/api/graphql/reference/_index.md @@ -1331,7 +1331,7 @@ four standard [pagination arguments](#pagination-arguments): | `topics` | [`[String!]`](#string) | Filter projects by topics. | | `trending` | [`Boolean`](#boolean) | Return only projects that are trending. | | `visibilityLevel` | [`VisibilityLevelsEnum`](#visibilitylevelsenum) | Filter projects by visibility level. | -| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`. | +| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. Requires `ids` to be sent. This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`. | | `withIssuesEnabled` | [`Boolean`](#boolean) | Return only projects with issues enabled. | | `withMergeRequestsEnabled` | [`Boolean`](#boolean) | Return only projects with merge requests enabled. | @@ -35638,7 +35638,7 @@ four standard [pagination arguments](#pagination-arguments): | `topics` | [`[String!]`](#string) | Filter projects by topics. | | `trending` | [`Boolean`](#boolean) | Return only projects that are trending. | | `visibilityLevel` | [`VisibilityLevelsEnum`](#visibilitylevelsenum) | Filter projects by visibility level. | -| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`. | +| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. Requires `ids` to be sent. This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`. | | `withIssuesEnabled` | [`Boolean`](#boolean) | Return only projects with issues enabled. | | `withMergeRequestsEnabled` | [`Boolean`](#boolean) | Return only projects with merge requests enabled. | diff --git a/ee/app/graphql/ee/resolvers/projects_resolver.rb b/ee/app/graphql/ee/resolvers/projects_resolver.rb index 6f5afc5476674c..e823ee559572e1 100644 --- a/ee/app/graphql/ee/resolvers/projects_resolver.rb +++ b/ee/app/graphql/ee/resolvers/projects_resolver.rb @@ -32,6 +32,13 @@ def finder_params(args) .merge(args.slice(:include_hidden, :with_code_embeddings_indexed)) .merge(filter_expired_saml_session_projects: true) end + + override :validate_args! + def validate_args!(args) + return unless args[:with_code_embeddings_indexed].present? && args[:ids].nil? + + raise ::Gitlab::Graphql::Errors::ArgumentError, 'with_code_embeddings_indexed should be only used with ids' + end end end end diff --git a/ee/app/models/ai/active_context/code/repository.rb b/ee/app/models/ai/active_context/code/repository.rb index ac581e70ce5047..55ce59f0bf0c7c 100644 --- a/ee/app/models/ai/active_context/code/repository.rb +++ b/ee/app/models/ai/active_context/code/repository.rb @@ -54,7 +54,6 @@ class Repository < ApplicationRecord joins(:active_context_connection).where(active_context_connection: { active: true }) } - scope :pending_with_active_connection, -> { by_state(:pending).with_active_connection } scope :ready_with_active_connection, -> { ready.with_active_connection } private diff --git a/ee/app/models/ee/project.rb b/ee/app/models/ee/project.rb index 57e7b23c4863ca..f93b65a06f5d2f 100644 --- a/ee/app/models/ee/project.rb +++ b/ee/app/models/ee/project.rb @@ -449,13 +449,14 @@ def lock_for_confirmation!(id) "AND zoekt_repositories.zoekt_index_id = ?)", index_id) } - scope :with_ready_active_context_code_repository, -> { - joins(:ready_active_context_code_repository) - } - # can't filter with id on projects because project_id is used as partitioning key # on p_ai_active_context_code_repositories, query by project_id enables partition pruning scope :with_ready_active_context_code_repository_project_ids, ->(project_ids) { + unless project_ids.present? + raise ArgumentError, "project_ids must be a non-empty array to enable " \ + "partition scan on active_context_code_repository" + end + joins(:ready_active_context_code_repository) .where(p_ai_active_context_code_repositories: { project_id: project_ids }) } diff --git a/ee/spec/finders/ee/projects_finder_spec.rb b/ee/spec/finders/ee/projects_finder_spec.rb index 90b047ec6bdc9a..2660efe7fd7901 100644 --- a/ee/spec/finders/ee/projects_finder_spec.rb +++ b/ee/spec/finders/ee/projects_finder_spec.rb @@ -140,6 +140,10 @@ ) end + let(:project_ids_relation) do + [ultimate_project.id, ultimate_project2.id, premium_project.id, no_plan_project.id] + end + context 'when ai_active_context_connection is inactive' do it 'returns no project' do is_expected.to be_empty diff --git a/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb b/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb index fc47681f4f5303..4f0489d88e3019 100644 --- a/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb +++ b/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb @@ -54,10 +54,14 @@ end it 'returns no projects' do - is_expected.to eq [] + resolve = resolve(described_class, obj: nil, args: filters, ctx: { current_user: user }) + expect(resolve).to be_a(Gitlab::Graphql::Errors::ArgumentError) + expect(resolve.message).to eq('with_code_embeddings_indexed should be only used with ids') end context 'when there are active code_embeddings_repository' do + let!(:filters) { { with_code_embeddings_indexed: true, ids: [project.to_global_id.to_s] } } + before do code_embeddings_repository.active_context_connection.update!(active: true) code_embeddings_repository.update!(state: :ready) diff --git a/ee/spec/models/ai/active_context/code/repository_spec.rb b/ee/spec/models/ai/active_context/code/repository_spec.rb index 207bbb694f1fb5..9657cb49ed37c0 100644 --- a/ee/spec/models/ai/active_context/code/repository_spec.rb +++ b/ee/spec/models/ai/active_context/code/repository_spec.rb @@ -158,28 +158,6 @@ end end - describe '.pending_with_active_connection' do - let_it_be(:active_connection) { create(:ai_active_context_connection) } - let_it_be(:inactive_connection) { create(:ai_active_context_connection, :inactive) } - let_it_be(:pending_with_active) do - create(:ai_active_context_code_repository, state: :pending, active_context_connection: active_connection) - end - - let_it_be(:ready_with_active) do - create(:ai_active_context_code_repository, state: :ready, active_context_connection: active_connection) - end - - let_it_be(:pending_with_inactive) do - create(:ai_active_context_code_repository, state: :pending, active_context_connection: inactive_connection) - end - - it 'returns pending repositories with active connections' do - result = described_class.pending_with_active_connection - - expect(result).to contain_exactly(pending_with_active) - end - end - describe '.ready_with_active_connection' do it 'does not return repositories with inactive connection' do expect(described_class.ready_with_active_connection).to be_empty diff --git a/ee/spec/models/ee/project_spec.rb b/ee/spec/models/ee/project_spec.rb index a8bf1f0cb35dd4..06cae8737d4856 100644 --- a/ee/spec/models/ee/project_spec.rb +++ b/ee/spec/models/ee/project_spec.rb @@ -856,7 +856,7 @@ end end - describe '.with_ready_active_context_code_repositories' do + describe '.with_ready_active_context_code_repository_project_ids' do let_it_be(:project) { create(:project) } let_it_be(:namespace) { create(:group) } let_it_be(:code_embeddings_enabled_namespace) do @@ -871,8 +871,20 @@ ) end + it 'raises ArgumentError when called without project_ids' do + expect do + described_class.with_ready_active_context_code_repository_project_ids(nil) + end.to raise_error(ArgumentError, /project_ids must be a non-empty array/) + end + + it 'raises ArgumentError when called with empty array' do + expect do + described_class.with_ready_active_context_code_repository_project_ids([]) + end.to raise_error(ArgumentError, /project_ids must be a non-empty array/) + end + it 'return no project' do - expect(described_class.with_ready_active_context_code_repository).to be_empty + expect(described_class.with_ready_active_context_code_repository_project_ids(project.id)).to be_empty end context "when embedding repository is ready" do @@ -881,7 +893,7 @@ end it 'returns projects ai_active_context_code_repositories ready' do - expect(described_class.with_ready_active_context_code_repository).to be_empty + expect(described_class.with_ready_active_context_code_repository_project_ids(project.id)).to be_empty end context "when embedding repository is ready" do @@ -890,7 +902,7 @@ end it 'returns projects ai_active_context_code_repositories ready' do - expect(described_class.with_ready_active_context_code_repository).to contain_exactly(project) + expect(described_class.with_ready_active_context_code_repository_project_ids(project.id)).to contain_exactly(project) end end end -- GitLab From 2c54c641b6050b15eff512fd0acf1441f2bb9c9a Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 10 Jul 2025 16:21:26 -0400 Subject: [PATCH 11/19] update description --- doc/api/graphql/reference/_index.md | 4 ++-- ee/app/graphql/ee/resolvers/projects_resolver.rb | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/api/graphql/reference/_index.md b/doc/api/graphql/reference/_index.md index 0a8b555ab21c6c..1e8a4e80df7dc6 100644 --- a/doc/api/graphql/reference/_index.md +++ b/doc/api/graphql/reference/_index.md @@ -1331,7 +1331,7 @@ four standard [pagination arguments](#pagination-arguments): | `topics` | [`[String!]`](#string) | Filter projects by topics. | | `trending` | [`Boolean`](#boolean) | Return only projects that are trending. | | `visibilityLevel` | [`VisibilityLevelsEnum`](#visibilitylevelsenum) | Filter projects by visibility level. | -| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. Requires `ids` to be sent. This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`. | +| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. Requires `ids` to be sent. Applies only if the feature flag `allow_with_code_embeddings_indexed_projects_filter` is enabled. | | `withIssuesEnabled` | [`Boolean`](#boolean) | Return only projects with issues enabled. | | `withMergeRequestsEnabled` | [`Boolean`](#boolean) | Return only projects with merge requests enabled. | @@ -35638,7 +35638,7 @@ four standard [pagination arguments](#pagination-arguments): | `topics` | [`[String!]`](#string) | Filter projects by topics. | | `trending` | [`Boolean`](#boolean) | Return only projects that are trending. | | `visibilityLevel` | [`VisibilityLevelsEnum`](#visibilitylevelsenum) | Filter projects by visibility level. | -| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. Requires `ids` to be sent. This is only applied with the FF `allow_with_code_embeddings_indexed_projects_filter=true`. | +| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. Requires `ids` to be sent. Applies only if the feature flag `allow_with_code_embeddings_indexed_projects_filter` is enabled. | | `withIssuesEnabled` | [`Boolean`](#boolean) | Return only projects with issues enabled. | | `withMergeRequestsEnabled` | [`Boolean`](#boolean) | Return only projects with merge requests enabled. | diff --git a/ee/app/graphql/ee/resolvers/projects_resolver.rb b/ee/app/graphql/ee/resolvers/projects_resolver.rb index e823ee559572e1..646db69a4dc876 100644 --- a/ee/app/graphql/ee/resolvers/projects_resolver.rb +++ b/ee/app/graphql/ee/resolvers/projects_resolver.rb @@ -15,8 +15,8 @@ module ProjectsResolver required: false, experiment: { milestone: '18.2' }, description: "Include projects with indexed code embeddings. " \ - "Requires `ids` to be sent. This is only applied with the FF " \ - "`allow_with_code_embeddings_indexed_projects_filter=true`." + "Requires `ids` to be sent. Applies only if the feature flag " \ + "`allow_with_code_embeddings_indexed_projects_filter` is enabled." before_connection_authorization do |projects, current_user| ::Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects, current_user).execute @@ -35,6 +35,8 @@ def finder_params(args) override :validate_args! def validate_args!(args) + super(args) + return unless args[:with_code_embeddings_indexed].present? && args[:ids].nil? raise ::Gitlab::Graphql::Errors::ArgumentError, 'with_code_embeddings_indexed should be only used with ids' -- GitLab From ab0aa0e7e9d406bb2026888590c30b86b152181e Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 10 Jul 2025 16:29:04 -0400 Subject: [PATCH 12/19] update description --- ee/spec/finders/ee/projects_finder_spec.rb | 10 +++++++++- ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ee/spec/finders/ee/projects_finder_spec.rb b/ee/spec/finders/ee/projects_finder_spec.rb index 2660efe7fd7901..172d059ddf6785 100644 --- a/ee/spec/finders/ee/projects_finder_spec.rb +++ b/ee/spec/finders/ee/projects_finder_spec.rb @@ -176,7 +176,15 @@ stub_feature_flags(allow_with_code_embeddings_indexed_projects_filter: false) end - it 'returns no project code_embeddings_repository' do + it 'projects are not filtered by with_code_embeddings_indexed' do + is_expected.to contain_exactly(ultimate_project, ultimate_project2, premium_project, no_plan_project) + end + end + + context 'when project_ids_relation is nil' do + let(:project_ids_relation) { nil } + + it 'projects are not filtered by with_code_embeddings_indexed' do is_expected.to contain_exactly(ultimate_project, ultimate_project2, premium_project, no_plan_project) end end diff --git a/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb b/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb index 4f0489d88e3019..f0cbadc9a8c78f 100644 --- a/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb +++ b/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb @@ -53,7 +53,7 @@ ) end - it 'returns no projects' do + it 'raises error when called with with_code_embeddings_indexed and without ids' do resolve = resolve(described_class, obj: nil, args: filters, ctx: { current_user: user }) expect(resolve).to be_a(Gitlab::Graphql::Errors::ArgumentError) expect(resolve.message).to eq('with_code_embeddings_indexed should be only used with ids') -- GitLab From cfe8156abd1b9d755482e23cdd121d530fb42986 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 10 Jul 2025 21:26:42 -0400 Subject: [PATCH 13/19] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Pam Artiaga --- ee/app/finders/ee/projects_finder.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ee/app/finders/ee/projects_finder.rb b/ee/app/finders/ee/projects_finder.rb index 7e313d0c7b94a8..d2eaa1ffb81f1e 100644 --- a/ee/app/finders/ee/projects_finder.rb +++ b/ee/app/finders/ee/projects_finder.rb @@ -51,7 +51,9 @@ def by_feature_available(collection) def by_code_embeddings_indexed(items) # project ids is required because active_context_code_repositories is a partitioned table # project_id is used to do the partition prune - return items unless project_ids_relation.present? + # additionally, we won't allow a relation because this would result in a nested query + # and could mean that partitioning pruning isn't done + return items if project_ids_relation.blank? || !project_ids.is_a?(Array) if ::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) && ::Feature.enabled?(:allow_with_code_embeddings_indexed_projects_filter, current_user) -- GitLab From 38ce4cb5f4a076decffc364678d4ba059e18194f Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 10 Jul 2025 21:58:22 -0400 Subject: [PATCH 14/19] update project_ids_relations --- ee/app/finders/ee/projects_finder.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/app/finders/ee/projects_finder.rb b/ee/app/finders/ee/projects_finder.rb index d2eaa1ffb81f1e..c34ec531b2cdad 100644 --- a/ee/app/finders/ee/projects_finder.rb +++ b/ee/app/finders/ee/projects_finder.rb @@ -53,7 +53,7 @@ def by_code_embeddings_indexed(items) # project_id is used to do the partition prune # additionally, we won't allow a relation because this would result in a nested query # and could mean that partitioning pruning isn't done - return items if project_ids_relation.blank? || !project_ids.is_a?(Array) + return items if project_ids_relation.blank? || !project_ids_relation.is_a?(Array) if ::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) && ::Feature.enabled?(:allow_with_code_embeddings_indexed_projects_filter, current_user) -- GitLab From 3566a1d8d58b4a6f1cde8878b642d665e60a57db Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Thu, 10 Jul 2025 23:42:31 -0400 Subject: [PATCH 15/19] update by code embeddings index filter --- ee/app/finders/ee/projects_finder.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ee/app/finders/ee/projects_finder.rb b/ee/app/finders/ee/projects_finder.rb index c34ec531b2cdad..c2ee5ad1b3553b 100644 --- a/ee/app/finders/ee/projects_finder.rb +++ b/ee/app/finders/ee/projects_finder.rb @@ -51,11 +51,8 @@ def by_feature_available(collection) def by_code_embeddings_indexed(items) # project ids is required because active_context_code_repositories is a partitioned table # project_id is used to do the partition prune - # additionally, we won't allow a relation because this would result in a nested query - # and could mean that partitioning pruning isn't done - return items if project_ids_relation.blank? || !project_ids_relation.is_a?(Array) - if ::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) && + project_ids_relation.present? && ::Feature.enabled?(:allow_with_code_embeddings_indexed_projects_filter, current_user) items.with_ready_active_context_code_repository_project_ids project_ids_relation else -- GitLab From e1510d2be93b582e9ece24c3159f358db03b5b23 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Fri, 11 Jul 2025 12:15:15 -0400 Subject: [PATCH 16/19] update by code embeddings index filter --- ee/app/finders/ee/projects_finder.rb | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ee/app/finders/ee/projects_finder.rb b/ee/app/finders/ee/projects_finder.rb index c2ee5ad1b3553b..f902223790f01c 100644 --- a/ee/app/finders/ee/projects_finder.rb +++ b/ee/app/finders/ee/projects_finder.rb @@ -51,13 +51,19 @@ def by_feature_available(collection) def by_code_embeddings_indexed(items) # project ids is required because active_context_code_repositories is a partitioned table # project_id is used to do the partition prune - if ::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) && - project_ids_relation.present? && - ::Feature.enabled?(:allow_with_code_embeddings_indexed_projects_filter, current_user) - items.with_ready_active_context_code_repository_project_ids project_ids_relation - else - items + if ::Feature.disabled?(:allow_with_code_embeddings_indexed_projects_filter, current_user) || + !::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) || + project_ids_relation.blank? || + !project_ids_relation.is_a?(Array) + return items end + + # only apply the code_embeddings_indexed filter when + # 1. feature flag `allow_with_code_embeddings_indexed_projects_filter` is enabled AND + # 2. params[:with_code_embeddings_indexed] is present and is true AND + # 3. project_ids_relation is an Array AND + # 4. project_ids_relation is not empty + items.with_ready_active_context_code_repository_project_ids project_ids_relation end def by_hidden(items) -- GitLab From 98baef6624153390ed45fae804819bad7bc7e6a6 Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Mon, 14 Jul 2025 07:32:24 -0400 Subject: [PATCH 17/19] update the with_code_embeddings_indexed description --- ee/app/finders/ee/projects_finder.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ee/app/finders/ee/projects_finder.rb b/ee/app/finders/ee/projects_finder.rb index f902223790f01c..4407659b871a86 100644 --- a/ee/app/finders/ee/projects_finder.rb +++ b/ee/app/finders/ee/projects_finder.rb @@ -15,7 +15,8 @@ module EE # active: boolean - Whether to include projects that are neither archived or marked # for deletion. # with_code_embeddings_indexed: boolean - Whether to include projects that have - # indexed embeddings for their code + # indexed embeddings for their code. This requires the `project_ids_relation` parameter, + # passed in as an integer array module ProjectsFinder include Gitlab::Auth::Saml::SsoSessionFilterable extend ::Gitlab::Utils::Override @@ -49,20 +50,19 @@ def by_feature_available(collection) end def by_code_embeddings_indexed(items) - # project ids is required because active_context_code_repositories is a partitioned table - # project_id is used to do the partition prune + # return original projects relation if `with_code_embeddings_indexed` is false or FF is disabled if ::Feature.disabled?(:allow_with_code_embeddings_indexed_projects_filter, current_user) || - !::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) || - project_ids_relation.blank? || - !project_ids_relation.is_a?(Array) + ::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) return items end - # only apply the code_embeddings_indexed filter when - # 1. feature flag `allow_with_code_embeddings_indexed_projects_filter` is enabled AND - # 2. params[:with_code_embeddings_indexed] is present and is true AND - # 3. project_ids_relation is an Array AND - # 4. project_ids_relation is not empty + # return empty project relation if `project_ids_relation` is invalid + # project ids is required because active_context_code_repositories is a partitioned table + # project_id is used to do the partition prune + # additionally, we won't allow a relation because this would result in a nested query + # and could mean that partitioning pruning isn't done + return Project.none if project_ids_relation.blank? || !project_ids_relation.is_a?(Array) + items.with_ready_active_context_code_repository_project_ids project_ids_relation end -- GitLab From 531b5f8703731be058d0257c17a8b3322e05747f Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Mon, 14 Jul 2025 08:22:47 -0400 Subject: [PATCH 18/19] update the with_code_embeddings_indexed description --- ee/app/finders/ee/projects_finder.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/app/finders/ee/projects_finder.rb b/ee/app/finders/ee/projects_finder.rb index 4407659b871a86..123f63428aa7f4 100644 --- a/ee/app/finders/ee/projects_finder.rb +++ b/ee/app/finders/ee/projects_finder.rb @@ -61,7 +61,7 @@ def by_code_embeddings_indexed(items) # project_id is used to do the partition prune # additionally, we won't allow a relation because this would result in a nested query # and could mean that partitioning pruning isn't done - return Project.none if project_ids_relation.blank? || !project_ids_relation.is_a?(Array) + return items.none if project_ids_relation.blank? || !project_ids_relation.is_a?(Array) items.with_ready_active_context_code_repository_project_ids project_ids_relation end -- GitLab From 97f6b73ee556112e211f109d8a61ded0709db09e Mon Sep 17 00:00:00 2001 From: tgao3701908 Date: Mon, 14 Jul 2025 08:42:35 -0400 Subject: [PATCH 19/19] add unit test to cover when project_ids_relation is an active_record_relation --- ee/app/finders/ee/projects_finder.rb | 2 +- ee/spec/finders/ee/projects_finder_spec.rb | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ee/app/finders/ee/projects_finder.rb b/ee/app/finders/ee/projects_finder.rb index 123f63428aa7f4..d8feb4079ec718 100644 --- a/ee/app/finders/ee/projects_finder.rb +++ b/ee/app/finders/ee/projects_finder.rb @@ -52,7 +52,7 @@ def by_feature_available(collection) def by_code_embeddings_indexed(items) # return original projects relation if `with_code_embeddings_indexed` is false or FF is disabled if ::Feature.disabled?(:allow_with_code_embeddings_indexed_projects_filter, current_user) || - ::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) + !::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) return items end diff --git a/ee/spec/finders/ee/projects_finder_spec.rb b/ee/spec/finders/ee/projects_finder_spec.rb index 172d059ddf6785..60b8354131e6d4 100644 --- a/ee/spec/finders/ee/projects_finder_spec.rb +++ b/ee/spec/finders/ee/projects_finder_spec.rb @@ -185,7 +185,15 @@ let(:project_ids_relation) { nil } it 'projects are not filtered by with_code_embeddings_indexed' do - is_expected.to contain_exactly(ultimate_project, ultimate_project2, premium_project, no_plan_project) + is_expected.to be_empty + end + end + + context 'when project_ids_relation is an active_record_relation' do + let(:project_ids_relation) { Project.where(id: [ultimate_project.id]) } + + it 'projects are not filtered by with_code_embeddings_indexed' do + is_expected.to be_empty end end end -- GitLab