diff --git a/doc/api/graphql/reference/_index.md b/doc/api/graphql/reference/_index.md index 462f91fff6ed29e488f74323ee3ee37ae4fe46a7..1e8a4e80df7dc66e81d1635a972427cd6e9b756f 100644 --- a/doc/api/graphql/reference/_index.md +++ b/doc/api/graphql/reference/_index.md @@ -1331,6 +1331,7 @@ four standard [pagination arguments](#pagination-arguments): | `topics` | [`[String!]`](#string) | Filter projects by topics. | | `trending` | [`Boolean`](#boolean) | Return only projects that are trending. | | `visibilityLevel` | [`VisibilityLevelsEnum`](#visibilitylevelsenum) | Filter projects by visibility level. | +| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. Requires `ids` to be sent. Applies only if the feature flag `allow_with_code_embeddings_indexed_projects_filter` is enabled. | | `withIssuesEnabled` | [`Boolean`](#boolean) | Return only projects with issues enabled. | | `withMergeRequestsEnabled` | [`Boolean`](#boolean) | Return only projects with merge requests enabled. | @@ -35637,6 +35638,7 @@ four standard [pagination arguments](#pagination-arguments): | `topics` | [`[String!]`](#string) | Filter projects by topics. | | `trending` | [`Boolean`](#boolean) | Return only projects that are trending. | | `visibilityLevel` | [`VisibilityLevelsEnum`](#visibilitylevelsenum) | Filter projects by visibility level. | +| `withCodeEmbeddingsIndexed` {{< icon name="warning-solid" >}} | [`Boolean`](#boolean) | **Introduced** in GitLab 18.2. **Status**: Experiment. Include projects with indexed code embeddings. Requires `ids` to be sent. Applies only if the feature flag `allow_with_code_embeddings_indexed_projects_filter` is enabled. | | `withIssuesEnabled` | [`Boolean`](#boolean) | Return only projects with issues enabled. | | `withMergeRequestsEnabled` | [`Boolean`](#boolean) | Return only projects with merge requests enabled. | diff --git a/ee/app/finders/ee/projects_finder.rb b/ee/app/finders/ee/projects_finder.rb index ee37ce1dbbe40db7bf2db0e914b77057ca657e84..d8feb4079ec718b5c19f0ac58c49617853323b7c 100644 --- a/ee/app/finders/ee/projects_finder.rb +++ b/ee/app/finders/ee/projects_finder.rb @@ -14,6 +14,9 @@ module EE # filter_expired_saml_session_projects: boolean # active: boolean - Whether to include projects that are neither archived or marked # for deletion. + # with_code_embeddings_indexed: boolean - Whether to include projects that have + # indexed embeddings for their code. This requires the `project_ids_relation` parameter, + # passed in as an integer array module ProjectsFinder include Gitlab::Auth::Saml::SsoSessionFilterable extend ::Gitlab::Utils::Override @@ -26,6 +29,7 @@ def filter_projects(collection) collection = by_plans(collection) collection = by_feature_available(collection) collection = by_hidden(collection) + collection = by_code_embeddings_indexed(collection) by_saml_sso_session(collection) end @@ -45,6 +49,23 @@ def by_feature_available(collection) end end + def by_code_embeddings_indexed(items) + # return original projects relation if `with_code_embeddings_indexed` is false or FF is disabled + if ::Feature.disabled?(:allow_with_code_embeddings_indexed_projects_filter, current_user) || + !::Gitlab::Utils.to_boolean(params[:with_code_embeddings_indexed]) + return items + end + + # return empty project relation if `project_ids_relation` is invalid + # project ids is required because active_context_code_repositories is a partitioned table + # project_id is used to do the partition prune + # additionally, we won't allow a relation because this would result in a nested query + # and could mean that partitioning pruning isn't done + return items.none if project_ids_relation.blank? || !project_ids_relation.is_a?(Array) + + items.with_ready_active_context_code_repository_project_ids project_ids_relation + end + def by_hidden(items) params[:include_hidden].present? ? items : items.not_hidden end diff --git a/ee/app/graphql/ee/resolvers/projects_resolver.rb b/ee/app/graphql/ee/resolvers/projects_resolver.rb index 75560a5d4d6b8758fe9ec43ab7d9052a360f19ab..646db69a4dc87676540da3045998d940356e62c6 100644 --- a/ee/app/graphql/ee/resolvers/projects_resolver.rb +++ b/ee/app/graphql/ee/resolvers/projects_resolver.rb @@ -11,6 +11,13 @@ module ProjectsResolver required: false, description: 'Include hidden projects.' + argument :with_code_embeddings_indexed, GraphQL::Types::Boolean, + required: false, + experiment: { milestone: '18.2' }, + description: "Include projects with indexed code embeddings. " \ + "Requires `ids` to be sent. Applies only if the feature flag " \ + "`allow_with_code_embeddings_indexed_projects_filter` is enabled." + before_connection_authorization do |projects, current_user| ::Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects, current_user).execute ::Preloaders::UserMemberRolesInProjectsPreloader.new(projects: projects, user: current_user).execute @@ -22,9 +29,18 @@ module ProjectsResolver override :finder_params def finder_params(args) super(args) - .merge(args.slice(:include_hidden)) + .merge(args.slice(:include_hidden, :with_code_embeddings_indexed)) .merge(filter_expired_saml_session_projects: true) end + + override :validate_args! + def validate_args!(args) + super(args) + + return unless args[:with_code_embeddings_indexed].present? && args[:ids].nil? + + raise ::Gitlab::Graphql::Errors::ArgumentError, 'with_code_embeddings_indexed should be only used with ids' + end end end end diff --git a/ee/app/models/ai/active_context/code/repository.rb b/ee/app/models/ai/active_context/code/repository.rb index c4e78586cf54eef276ee621a395ce2ff2f0b4781..55ce59f0bf0c7c3d4bd0924bd408a02b67b0f5c9 100644 --- a/ee/app/models/ai/active_context/code/repository.rb +++ b/ee/app/models/ai/active_context/code/repository.rb @@ -54,6 +54,8 @@ class Repository < ApplicationRecord joins(:active_context_connection).where(active_context_connection: { active: true }) } + scope :ready_with_active_connection, -> { ready.with_active_connection } + private def set_last_commit diff --git a/ee/app/models/ee/project.rb b/ee/app/models/ee/project.rb index 2c719db8d2bd7b93fe18e7a1647cdd20bea30f4f..f93b65a06f5d2f8ea81a56eb83c8bff1751fb62f 100644 --- a/ee/app/models/ee/project.rb +++ b/ee/app/models/ee/project.rb @@ -232,6 +232,10 @@ def lock_for_confirmation!(id) has_many :configured_ai_catalog_items, class_name: '::Ai::Catalog::ItemConsumer', inverse_of: :project + has_one :ready_active_context_code_repository, + -> { ready_with_active_connection }, + class_name: 'Ai::ActiveContext::Code::Repository' + elastic_index_dependant_association :issues, on_change: :visibility_level elastic_index_dependant_association :issues, on_change: :archived elastic_index_dependant_association :work_items, on_change: :visibility_level @@ -445,6 +449,18 @@ def lock_for_confirmation!(id) "AND zoekt_repositories.zoekt_index_id = ?)", index_id) } + # can't filter with id on projects because project_id is used as partitioning key + # on p_ai_active_context_code_repositories, query by project_id enables partition pruning + scope :with_ready_active_context_code_repository_project_ids, ->(project_ids) { + unless project_ids.present? + raise ArgumentError, "project_ids must be a non-empty array to enable " \ + "partition scan on active_context_code_repository" + end + + joins(:ready_active_context_code_repository) + .where(p_ai_active_context_code_repositories: { project_id: project_ids }) + } + delegate :shared_runners_seconds, to: :statistics, allow_nil: true delegate :ci_minutes_usage, to: :shared_runners_limit_namespace diff --git a/ee/config/feature_flags/experiment/allow_with_code_embeddings_indexed_projects_filter.yml b/ee/config/feature_flags/experiment/allow_with_code_embeddings_indexed_projects_filter.yml new file mode 100644 index 0000000000000000000000000000000000000000..9ab027d074c290f8c65fafe5e86baf3747ae032e --- /dev/null +++ b/ee/config/feature_flags/experiment/allow_with_code_embeddings_indexed_projects_filter.yml @@ -0,0 +1,10 @@ +--- +name: allow_with_code_embeddings_indexed_projects_filter +description: +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/547112 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/195699 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/553819 +milestone: '18.2' +group: group::code creation +type: experiment +default_enabled: false diff --git a/ee/spec/finders/ee/projects_finder_spec.rb b/ee/spec/finders/ee/projects_finder_spec.rb index d1d86e7d02d420cebb72e6175edc70a2a7ef5806..60b8354131e6d43bdde22a2503c2dce53dff12e3 100644 --- a/ee/spec/finders/ee/projects_finder_spec.rb +++ b/ee/spec/finders/ee/projects_finder_spec.rb @@ -108,6 +108,98 @@ end end + context 'filter by code_embeddings_indexed' do + let_it_be(:params) { { with_code_embeddings_indexed: true } } + + let_it_be(:namespace) { create(:group) } + let!(:code_embeddings_enabled_namespace) do + create(:ai_active_context_code_enabled_namespace, namespace: namespace) + end + + let!(:code_embeddings_repository_1) do + create( + :ai_active_context_code_repository, + project: ultimate_project, + enabled_namespace: code_embeddings_enabled_namespace + ) + end + + let!(:code_embeddings_repository_2) do + create( + :ai_active_context_code_repository, + project: ultimate_project2, + enabled_namespace: code_embeddings_enabled_namespace + ) + end + + let!(:code_embeddings_repository_3) do + create( + :ai_active_context_code_repository, + project: premium_project, + enabled_namespace: code_embeddings_enabled_namespace + ) + end + + let(:project_ids_relation) do + [ultimate_project.id, ultimate_project2.id, premium_project.id, no_plan_project.id] + end + + context 'when ai_active_context_connection is inactive' do + it 'returns no project' do + is_expected.to be_empty + end + end + + context 'when ai_active_context_connection is active' do + before do + code_embeddings_enabled_namespace.active_context_connection.reload.update!(active: true) + end + + context 'when code_embeddings_repository is not ready' do + it 'returns no project code_embeddings_repository' do + is_expected.to be_empty + end + end + + context 'when code_embeddings_repository are ready' do + before do + code_embeddings_repository_1.update!(state: :ready) + code_embeddings_repository_3.update!(state: :ready) + end + + it 'returns project with ready code_embeddings_repository' do + is_expected.to contain_exactly(ultimate_project, premium_project) + end + + context 'when ff allow_with_code_embeddings_indexed_projects_filter is false' do + before do + stub_feature_flags(allow_with_code_embeddings_indexed_projects_filter: false) + end + + it 'projects are not filtered by with_code_embeddings_indexed' do + is_expected.to contain_exactly(ultimate_project, ultimate_project2, premium_project, no_plan_project) + end + end + + context 'when project_ids_relation is nil' do + let(:project_ids_relation) { nil } + + it 'projects are not filtered by with_code_embeddings_indexed' do + is_expected.to be_empty + end + end + + context 'when project_ids_relation is an active_record_relation' do + let(:project_ids_relation) { Project.where(id: [ultimate_project.id]) } + + it 'projects are not filtered by with_code_embeddings_indexed' do + is_expected.to be_empty + end + end + end + end + end + private def create_project(plan, visibility = :public) diff --git a/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb b/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb index 6ee03fe1d3f30f7090b09849d3e35eb04da60acd..f0cbadc9a8c78f06d954424c320fdd8ca68cee3d 100644 --- a/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb +++ b/ee/spec/graphql/ee/resolvers/projects_resolver_spec.rb @@ -36,5 +36,41 @@ it { is_expected.to contain_exactly(project, project_marked_for_deletion) } end + + context 'when requesting with_code_embeddings_indexed' do + let!(:filters) { { with_code_embeddings_indexed: true } } + + let_it_be(:namespace) { create(:group) } + let_it_be(:code_embeddings_enabled_namespace) do + create(:ai_active_context_code_enabled_namespace, namespace: namespace) + end + + let!(:code_embeddings_repository) do + create( + :ai_active_context_code_repository, + project: project, + enabled_namespace: code_embeddings_enabled_namespace + ) + end + + it 'raises error when called with with_code_embeddings_indexed and without ids' do + resolve = resolve(described_class, obj: nil, args: filters, ctx: { current_user: user }) + expect(resolve).to be_a(Gitlab::Graphql::Errors::ArgumentError) + expect(resolve.message).to eq('with_code_embeddings_indexed should be only used with ids') + end + + context 'when there are active code_embeddings_repository' do + let!(:filters) { { with_code_embeddings_indexed: true, ids: [project.to_global_id.to_s] } } + + before do + code_embeddings_repository.active_context_connection.update!(active: true) + code_embeddings_repository.update!(state: :ready) + end + + it 'returns projects with active code_embeddings_repository' do + is_expected.to contain_exactly(project) + end + end + end end end diff --git a/ee/spec/models/ai/active_context/code/repository_spec.rb b/ee/spec/models/ai/active_context/code/repository_spec.rb index 2bdd7e6a1cfc4e3f24ff5c0ea35ae2a7f51981e4..9657cb49ed37c0484d6a859341718a3fb1eab4f9 100644 --- a/ee/spec/models/ai/active_context/code/repository_spec.rb +++ b/ee/spec/models/ai/active_context/code/repository_spec.rb @@ -157,6 +157,32 @@ expect(result).to contain_exactly(repository_with_active_connection) end end + + describe '.ready_with_active_connection' do + it 'does not return repositories with inactive connection' do + expect(described_class.ready_with_active_connection).to be_empty + end + + context 'when connection is active' do + before do + repository.active_context_connection.update!(active: true) + end + + it 'returns repositories with active connection' do + expect(described_class.ready_with_active_connection).to be_empty + end + + context 'when connection is active and repository is ready' do + before do + repository.update!(state: :ready) + end + + it 'returns repositories with active connection' do + expect(described_class.ready_with_active_connection).to contain_exactly(repository) + end + end + end + end end describe 'table partitioning' do diff --git a/ee/spec/models/ee/project_spec.rb b/ee/spec/models/ee/project_spec.rb index fabadca96af40d7cd0d2add38a45262643ccddbe..06cae8737d485627e52a4899caab4ceb9677a875 100644 --- a/ee/spec/models/ee/project_spec.rb +++ b/ee/spec/models/ee/project_spec.rb @@ -855,6 +855,58 @@ expect(described_class.without_security_setting).to match_array([project_without_security_setting]) end end + + describe '.with_ready_active_context_code_repository_project_ids' do + let_it_be(:project) { create(:project) } + let_it_be(:namespace) { create(:group) } + let_it_be(:code_embeddings_enabled_namespace) do + create(:ai_active_context_code_enabled_namespace, namespace: namespace) + end + + let(:code_embeddings_repository) do + create( + :ai_active_context_code_repository, + project: project, + enabled_namespace: code_embeddings_enabled_namespace + ) + end + + it 'raises ArgumentError when called without project_ids' do + expect do + described_class.with_ready_active_context_code_repository_project_ids(nil) + end.to raise_error(ArgumentError, /project_ids must be a non-empty array/) + end + + it 'raises ArgumentError when called with empty array' do + expect do + described_class.with_ready_active_context_code_repository_project_ids([]) + end.to raise_error(ArgumentError, /project_ids must be a non-empty array/) + end + + it 'return no project' do + expect(described_class.with_ready_active_context_code_repository_project_ids(project.id)).to be_empty + end + + context "when embedding repository is ready" do + before do + code_embeddings_repository.update!(state: :ready) + end + + it 'returns projects ai_active_context_code_repositories ready' do + expect(described_class.with_ready_active_context_code_repository_project_ids(project.id)).to be_empty + end + + context "when embedding repository is ready" do + before do + code_embeddings_repository.active_context_connection.update!(active: true) + end + + it 'returns projects ai_active_context_code_repositories ready' do + expect(described_class.with_ready_active_context_code_repository_project_ids(project.id)).to contain_exactly(project) + end + end + end + end end describe 'validations' do diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 1e4ab2a91b1e4d9b54438d96a737628f19206cf0..e5a0bf6cd11ade71cb6ed1d83e9edff8229d7745 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -1374,6 +1374,7 @@ ee: excluded_attributes: project: - :vulnerability_hooks_integrations + - :ready_active_context_code_repository approval_rules: - :created_at - :updated_at diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index f010aeaa5e485f8b13aaa36be89e859a6b868797..bdc4c4aed44ca5a255f25a4e6b19598a8affcff4 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -618,6 +618,7 @@ project: - alert_hooks_integrations - incident_hooks_integrations - vulnerability_hooks_integrations +- ready_active_context_code_repository - apple_app_store_integration - google_cloud_platform_artifact_registry_integration - google_cloud_platform_workload_identity_federation_integration