diff --git a/CHANGELOG b/CHANGELOG index 0c71aab294f4eb0f83a966b0d9d9d6a8cde5dd92..e95e0226441779f5181ad8be0c000516dc0ecebc 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -98,6 +98,7 @@ v 8.5.8 v 8.5.7 - Bump Git version requirement to 2.7.3 + - Add ability to sync to remote mirrors. v 8.5.6 - Obtain a lease before querying LDAP diff --git a/app/assets/javascripts/dispatcher.js.coffee b/app/assets/javascripts/dispatcher.js.coffee index 9e7f3d25c072a8442d40088354bd0c1226676443..8d284f1bee5a7c71d04770d93e41d81b4e6be817 100644 --- a/app/assets/javascripts/dispatcher.js.coffee +++ b/app/assets/javascripts/dispatcher.js.coffee @@ -109,6 +109,7 @@ class Dispatcher new GroupsSelect() when 'projects:mirrors:show', 'projects:mirrors:update' new UsersSelect() + new MirrorUrlProtector() when 'admin:emails:show' new AdminEmailSelect() diff --git a/app/assets/javascripts/mirror_url_protector.js.coffee b/app/assets/javascripts/mirror_url_protector.js.coffee new file mode 100644 index 0000000000000000000000000000000000000000..1e7f93d201e9dc8751d07c03926813630b5beb0b --- /dev/null +++ b/app/assets/javascripts/mirror_url_protector.js.coffee @@ -0,0 +1,15 @@ +class @MirrorUrlProtector + constructor: -> + $('.toggle-remote-credentials').on 'click', (e) => + e.preventDefault() + + anchor = $(e.target) + inputUrl = anchor.prev() + + switch anchor.text() + when 'Show credentials' + anchor.text('Hide credentials') + inputUrl.val(inputUrl.data('full-url')) + when 'Hide credentials' + anchor.text('Show credentials') + inputUrl.val(inputUrl.data('safe-url')) diff --git a/app/assets/stylesheets/pages/projects.scss b/app/assets/stylesheets/pages/projects.scss index 71bde1174eeaf62d2e6f2651ff22ea6612bae9cb..8992470518098fedd2a1a525b7c488ab84fd4a41 100644 --- a/app/assets/stylesheets/pages/projects.scss +++ b/app/assets/stylesheets/pages/projects.scss @@ -461,7 +461,7 @@ pre.light-well { .cannot-be-merged, .cannot-be-merged:hover { - color: #e62958; + color: $error-exclamation-point; margin-top: 2px; } @@ -474,3 +474,7 @@ pre.light-well { color: #fff; } } + +.disabled-item { + @extend .btn.disabled; +} diff --git a/app/controllers/projects/mirrors_controller.rb b/app/controllers/projects/mirrors_controller.rb index 4bd7664a273af04544ec885340e815b1c2bc9f63..eb84c045f27e78cd554781cd1170a03c8c9f3d4e 100644 --- a/app/controllers/projects/mirrors_controller.rb +++ b/app/controllers/projects/mirrors_controller.rb @@ -2,6 +2,7 @@ class Projects::MirrorsController < Projects::ApplicationController # Authorize before_action :authorize_admin_project!, except: [:update_now] before_action :authorize_push_code!, only: [:update_now] + before_action :remote_mirror, only: [:show, :update] layout "project_settings" @@ -34,9 +35,21 @@ def update_now redirect_back_or_default(default: namespace_project_path(@project.namespace, @project)) end + def update_remote_now + @project.update_remote_mirrors + + flash[:notice] = "The remote repository is being updated..." + redirect_back_or_default(default: namespace_project_path(@project.namespace, @project)) + end + private + def remote_mirror + @remote_mirror = @project.remote_mirrors.first_or_initialize + end + def mirror_params - params.require(:project).permit(:mirror, :import_url, :mirror_user_id, :mirror_trigger_builds) + params.require(:project).permit(:mirror, :import_url, :mirror_user_id, :mirror_trigger_builds, + remote_mirrors_attributes: [:url, :id, :enabled]) end end diff --git a/app/models/project.rb b/app/models/project.rb index b13997caf28c2080370f7b6e1cdbb4419b67d1be..03bcf41bffd416e5404732f5bd1a445a9715996b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -2,41 +2,54 @@ # # Table name: projects # -# id :integer not null, primary key -# name :string(255) -# path :string(255) -# description :text -# created_at :datetime -# updated_at :datetime -# creator_id :integer -# issues_enabled :boolean default(TRUE), not null -# wall_enabled :boolean default(TRUE), not null -# merge_requests_enabled :boolean default(TRUE), not null -# wiki_enabled :boolean default(TRUE), not null -# namespace_id :integer -# issues_tracker :string(255) default("gitlab"), not null -# issues_tracker_id :string(255) -# snippets_enabled :boolean default(TRUE), not null -# last_activity_at :datetime -# import_url :string(255) -# visibility_level :integer default(0), not null -# archived :boolean default(FALSE), not null -# avatar :string(255) -# import_status :string(255) -# repository_size :float default(0.0) -# star_count :integer default(0), not null -# import_type :string(255) -# import_source :string(255) -# commit_count :integer default(0) -# import_error :text -# ci_id :integer -# builds_enabled :boolean default(TRUE), not null -# shared_runners_enabled :boolean default(TRUE), not null -# runners_token :string -# build_coverage_regex :string -# build_allow_git_fetch :boolean default(TRUE), not null -# build_timeout :integer default(3600), not null -# pending_delete :boolean +# id :integer not null, primary key +# name :string(255) +# path :string(255) +# description :text +# created_at :datetime +# updated_at :datetime +# creator_id :integer +# issues_enabled :boolean default(TRUE), not null +# wall_enabled :boolean default(TRUE), not null +# merge_requests_enabled :boolean default(TRUE), not null +# wiki_enabled :boolean default(TRUE), not null +# namespace_id :integer +# issues_tracker :string(255) default("gitlab"), not null +# issues_tracker_id :string(255) +# snippets_enabled :boolean default(TRUE), not null +# last_activity_at :datetime +# import_url :string(255) +# visibility_level :integer default(0), not null +# archived :boolean default(FALSE), not null +# avatar :string(255) +# import_status :string(255) +# repository_size :float default(0.0) +# star_count :integer default(0), not null +# import_type :string(255) +# import_source :string(255) +# commit_count :integer default(0) +# import_error :text +# ci_id :integer +# builds_enabled :boolean default(TRUE), not null +# shared_runners_enabled :boolean default(TRUE), not null +# runners_token :string +# build_coverage_regex :string +# build_allow_git_fetch :boolean default(TRUE), not null +# build_timeout :integer default(3600), not null +# pending_delete :boolean default(FALSE) +# public_builds :boolean default(TRUE), not null +# merge_requests_template :text +# merge_requests_rebase_enabled :boolean default(FALSE) +# approvals_before_merge :integer default(0), not null +# reset_approvals_on_push :boolean default(TRUE) +# merge_requests_ff_only_enabled :boolean default(FALSE) +# issues_template :text +# mirror :boolean default(FALSE), not null +# mirror_last_update_at :datetime +# mirror_last_successful_update_at :datetime +# mirror_user_id :integer +# mirror_trigger_builds :boolean default(FALSE), not null +# main_language :string # require 'carrierwave/orm/activerecord' @@ -76,18 +89,8 @@ def set_last_activity_at after_destroy :remove_pages - # update visibility_level of forks after_update :update_forks_visibility_level - def update_forks_visibility_level - return unless visibility_level < visibility_level_was - - forks.each do |forked_project| - if forked_project.visibility_level > visibility_level - forked_project.visibility_level = visibility_level - forked_project.save! - end - end - end + after_update :remove_mirror_repository_reference, if: :import_url_changed? ActsAsTaggableOn.strict_case_match = true acts_as_taggable_on :tags @@ -175,8 +178,11 @@ def update_forks_visibility_level has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, dependent: :destroy, class_name: 'Ci::Variable', foreign_key: :gl_project_id has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :gl_project_id + has_many :remote_mirrors, dependent: :destroy accepts_nested_attributes_for :variables, allow_destroy: true + accepts_nested_attributes_for :remote_mirrors, + allow_destroy: true, reject_if: proc { |attrs| attrs[:id].blank? && attrs[:url].blank? } delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true @@ -204,6 +210,7 @@ def update_forks_visibility_level url: { protocols: %w(ssh git http https) }, if: :external_import? validates :import_url, presence: true, if: :mirror? + validate :import_url_availability, if: :import_url_changed? validates :mirror_user, presence: true, if: :mirror? validates :star_count, numericality: { greater_than_or_equal_to: 0 } validate :check_limit, on: :create @@ -216,6 +223,7 @@ def update_forks_visibility_level add_authentication_token_field :runners_token before_save :ensure_runners_token + before_validation :mark_remote_mirrors_for_removal mount_uploader :avatar, AvatarUploader @@ -234,6 +242,7 @@ def update_forks_visibility_level scope :non_archived, -> { where(archived: false) } scope :mirror, -> { where(mirror: true) } scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } + scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct } state_machine :import_status, initial: :none do event :import_start do @@ -522,6 +531,23 @@ def update_mirror end end + def mark_import_as_failed(error_message) + import_fail + update_column(:import_error, error_message) + end + + def remote_mirror? + remote_mirrors.enabled.exists? + end + + def updating_remote_mirror? + remote_mirrors.enabled.started.exists? + end + + def update_remote_mirrors + remote_mirrors.each(&:sync) + end + def fetch_mirror return unless mirror? @@ -1192,4 +1218,31 @@ def merge_method=(method) def ff_merge_must_be_possible? self.merge_requests_ff_only_enabled || self.merge_requests_rebase_enabled end + + private + + def remove_mirror_repository_reference + repository.remove_remote(Repository::MIRROR_REMOTE) + end + + def update_forks_visibility_level + return unless visibility_level < visibility_level_was + + forks.each do |forked_project| + if forked_project.visibility_level > visibility_level + forked_project.visibility_level = visibility_level + forked_project.save! + end + end + end + + def import_url_availability + if remote_mirrors.find_by(url: import_url) + errors.add(:import_url, 'is already in use by the remote mirror') + end + end + + def mark_remote_mirrors_for_removal + remote_mirrors.each(&:mark_for_delete_if_blank_url) + end end diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb new file mode 100644 index 0000000000000000000000000000000000000000..d1f97b534b7bdb507f974ccad2530548859851dd --- /dev/null +++ b/app/models/remote_mirror.rb @@ -0,0 +1,149 @@ +# == Schema Information +# +# Table name: remote_mirrors +# +# id :integer not null, primary key +# project_id :integer +# url :string +# last_update_at :datetime +# last_error :string +# created_at :datetime not null +# updated_at :datetime not null +# last_successful_update_at :datetime +# update_status :string +# enabled :boolean default(TRUE) +# + +class RemoteMirror < ActiveRecord::Base + include AfterCommitQueue + + attr_encrypted :credentials, key: Gitlab::Application.secrets.db_key_base, marshal: true, encode: true, mode: :per_attribute_iv_and_salt + + belongs_to :project + + validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }, on: :create + validate :url_availability, if: :url_changed? + + after_save :refresh_remote, if: :url_changed? + after_update :reset_fields, if: :url_changed? + after_destroy :remove_remote + + scope :enabled, -> { where(enabled: true) } + scope :started, -> { with_update_status(:started) } + + state_machine :update_status, initial: :none do + event :update_start do + transition [:none, :finished] => :started + end + + event :update_finish do + transition started: :finished + end + + event :update_fail do + transition started: :failed + end + + event :update_retry do + transition failed: :started + end + + state :started + state :finished + state :failed + + after_transition any => :started, do: :schedule_update_job + + after_transition started: :finished do |remote_mirror, transaction| + timestamp = DateTime.now + remote_mirror.update_attributes!( + last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil + ) + end + + after_transition started: :failed do |remote_mirror, transaction| + remote_mirror.update(last_update_at: DateTime.now) + end + end + + def ref_name + "remote_mirror_#{id}" + end + + def update_failed? + update_status == 'failed' + end + + def update_in_progress? + update_status == 'started' + end + + def sync + return if !enabled || update_in_progress? + + update_failed? ? update_retry : update_start + end + + def mark_for_delete_if_blank_url + mark_for_destruction if url.blank? + end + + def mark_as_failed(error_message) + update_fail + update_column(:last_error, error_message) + end + + def url=(value) + mirror_url = Gitlab::ImportUrl.new(value) + + # Update credentials only if passed URL is different than the previous one. + self.credentials = mirror_url.credentials if url != value + + super(mirror_url.sanitized_url) + end + + def full_url + mirror_url = Gitlab::ImportUrl.new(url, credentials: credentials) + mirror_url.full_url + end + + def has_credentials? + credentials.values.any? + end + + private + + def url_availability + if project.import_url == url + errors.add(:url, 'is already in use') + end + end + + def reset_fields + update_columns( + last_error: nil, + last_update_at: nil, + last_successful_update_at: nil, + update_status: 'finished' + ) + end + + def schedule_update_job + run_after_commit(:add_update_job) + end + + def add_update_job + if project.repository_exists? + RepositoryUpdateRemoteMirrorWorker.perform_async(self.id) + end + end + + def refresh_remote + project.repository.remove_remote(ref_name) + project.repository.add_remote(ref_name, full_url) + end + + def remove_remote + project.repository.remove_remote(ref_name) + end +end diff --git a/app/models/repository.rb b/app/models/repository.rb index 4fee723e22ea4e59b28d2dbe9d1e3e266152743d..a1cf5ad2b945918a9bab2df7096f90a8e6283fc1 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -183,12 +183,27 @@ def rm_tag(tag_name) gitlab_shell.rm_tag(path_with_namespace, tag_name) end + def push_branches(project_name, remote, branch_names) + gitlab_shell.push_branches(project_name, remote, branch_names) + end + + def delete_remote_branches(project_name, remote, branch_names) + gitlab_shell.delete_remote_branches(project_name, remote, branch_names) + end + def add_remote(name, url) raw_repository.remote_add(name, url) rescue Rugged::ConfigError raw_repository.remote_update(name, url: url) end + def remove_remote(name) + raw_repository.remote_delete(name) + true + rescue Rugged::ConfigError + false + end + def set_remote_as_mirror(name) remote_config = raw_repository.rugged.config @@ -654,6 +669,14 @@ def branches @branches ||= raw_repository.branches end + def local_branches + branches_from_ref('heads') + end + + def remote_branches(remote_name) + branches_from_ref("remotes/#{remote_name}") + end + def tags @tags ||= raw_repository.tags end @@ -828,9 +851,9 @@ def fetch_geo_mirror(url) fetch_remote_forced!(Repository::MIRROR_GEO) end - def upstream_branches - rugged.references.each("refs/remotes/#{Repository::MIRROR_REMOTE}/*").map do |ref| - name = ref.name.sub(/\Arefs\/remotes\/#{Repository::MIRROR_REMOTE}\//, "") + def branches_from_ref(ref_name) + rugged.references.each("refs/#{ref_name}/*").map do |ref| + name = ref.name.sub(/\Arefs\/#{ref_name}\//, "") begin Gitlab::Git::Branch.new(name, ref.target) @@ -840,6 +863,10 @@ def upstream_branches end.compact end + def upstream_branches + branches_from_ref("remotes/#{Repository::MIRROR_REMOTE}") + end + def diverged_from_upstream?(branch_name) branch_commit = commit(branch_name) upstream_commit = commit("refs/remotes/#{MIRROR_REMOTE}/#{branch_name}") @@ -851,6 +878,17 @@ def diverged_from_upstream?(branch_name) end end + def upstream_has_diverged?(branch_name, remote_ref) + branch_commit = commit(branch_name) + upstream_commit = commit("refs/remotes/#{remote_ref}/#{branch_name}") + + if upstream_commit + !is_ancestor?(upstream_commit.id, branch_commit.id) + else + false + end + end + def up_to_date_with_upstream?(branch_name) branch_commit = commit(branch_name) upstream_commit = commit("refs/remotes/#{MIRROR_REMOTE}/#{branch_name}") diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..13a9265f28223304a2c27848b910de48783f909e --- /dev/null +++ b/app/services/projects/update_remote_mirror_service.rb @@ -0,0 +1,89 @@ +module Projects + class UpdateRemoteMirrorService < BaseService + attr_reader :mirror, :errors + + def execute(remote_mirror) + @mirror = remote_mirror + @errors = [] + + begin + repository.fetch_remote(mirror.ref_name) + + if divergent_branches.present? + errors << "The following branches have diverged from their local counterparts: #{divergent_branches.to_sentence}" + end + + push_to_mirror if changed_branches.present? + delete_from_mirror if deleted_branches.present? + rescue Gitlab::Shell::Error => e + errors << e.message.strip + end + + if errors.present? + error(errors.join("\n\n")) + else + success + end + end + + private + + def changed_branches + @changed_branches ||= local_branches.each_with_object([]) do |(name, branch), branches| + remote_branch = remote_branches[name] + + if remote_branch.nil? + branches << name + elsif branch.target == remote_branch.target + # Already up to date + elsif !repository.upstream_has_diverged?(name, mirror.ref_name) + branches << name + end + end + end + + def deleted_branches + @deleted_branches ||= remote_branches.each_with_object([]) do |(name, branch), branches| + local_branch = local_branches[name] + + if local_branch.nil? && project.commit(branch.target) + branches << name + end + end + end + + def push_to_mirror + default_branch, branches = changed_branches.partition { |name| project.default_branch == name } + + # Push the default branch first so it works fine when remote mirror is empty. + branches.unshift(*default_branch) + + repository.push_branches(project.path_with_namespace, mirror.ref_name, branches) + end + + def delete_from_mirror + repository.delete_remote_branches(project.path_with_namespace, mirror.ref_name, deleted_branches) + end + + def local_branches + @local_branches ||= repository.local_branches.each_with_object({}) do |branch, branches| + branches[branch.name] = branch + end + end + + def remote_branches + @remote_branches ||= repository.remote_branches(mirror.ref_name).each_with_object({}) do |branch, branches| + branches[branch.name] = branch + end + end + + def divergent_branches + remote_branches.each_with_object([]) do |(name, branch), branches| + if local_branches[name] && repository.upstream_has_diverged?(name, mirror.ref_name) + branches << name + end + end + end + + end +end diff --git a/app/views/projects/buttons/_update_mirror.html.haml b/app/views/projects/buttons/_update_mirror.html.haml index da93f25b8f5f926287a94110500fd6c8d973a27f..6157dd1beb064389a0e0bb71870ac3b185f65563 100644 --- a/app/views/projects/buttons/_update_mirror.html.haml +++ b/app/views/projects/buttons/_update_mirror.html.haml @@ -1,8 +1,31 @@ -- if @project.mirror? && can?(current_user, :push_code, @project) - - size = nil unless defined?(size) && size - - if @project.updating_mirror? - %span.btn.disabled.update-mirror-button.has-tooltip{title: "Updating from upstream..."} - = icon('refresh') - - else - = link_to update_now_namespace_project_mirror_path(@project.namespace, @project), method: :post, class: "btn update-mirror-button has-tooltip", title: "Update from upstream" do - = icon('refresh') +- if can?(current_user, :push_code, @project) + - if !@project.remote_mirror? && @project.mirror? + - size = nil unless defined?(size) && size + - if @project.updating_mirror? + %span.btn.disabled.update-mirror-button.has_tooltip{title: "Updating from upstream..."} + = icon('refresh') + - else + = link_to update_now_namespace_project_mirror_path(@project.namespace, @project), method: :post, class: "btn update-mirror-button has_tooltip", title: "Update from upstream" do + = icon('refresh') + - elsif @project.remote_mirror? && !@project.mirror? + - if @project.updating_remote_mirror? + %span.btn.disabled.update-mirror-button.has_tooltip{title: "Updating remote repository..."} + = icon('refresh') + - else + = link_to update_remote_now_namespace_project_mirror_path(@project.namespace, @project), method: :post, class: "btn update-mirror-button has_tooltip", title: "Update remote repository" do + = icon('refresh') + - elsif @project.remote_mirror? && @project.mirror? + .btn-group + %a.btn.dropdown-toggle{href: '#', 'data-toggle' => 'dropdown'} + = icon('refresh') + %ul.dropdown-menu.dropdown-menu-right + %li + - if @project.updating_mirror? + %a.disabled-item Updating from upstream... + - else + = link_to "Update this repository", update_now_namespace_project_mirror_path(@project.namespace, @project), method: :post + %li + - if @project.updating_remote_mirror? + %a.disabled-item Updating remote repository... + - else + = link_to "Update remote repository", update_remote_now_namespace_project_mirror_path(@project.namespace, @project), method: :post diff --git a/app/views/projects/mirrors/show.html.haml b/app/views/projects/mirrors/show.html.haml index 1144b776a9877a1716554d86923a2e091f232279..470fe5d3e479b7881ff66bc392c0e69338695f8b 100644 --- a/app/views/projects/mirrors/show.html.haml +++ b/app/views/projects/mirrors/show.html.haml @@ -1,27 +1,17 @@ - page_title "Mirror Repository" -.pull-right - - if @project.mirror_last_update_success? - Successfully updated #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}. - %span.prepend-left-10 - = render "shared/mirror_update_button" -%h3.page-title +%h3.page_title Mirror Repository %p.light - Set up your project to automatically have its branches, tags, and commits updated from an upstream repository every hour. + A repository can be setup as a mirror of another repository, and can also have a remote mirror associated. -%hr.clearfix +%p.light + When the repository is configured as a mirror, all of its content will automatically be updated from the repository configured in the Pull from a remote repository section. -- if @project.mirror_last_update_failed? - .panel.panel-danger - .panel-heading - The repository failed to update #{time_ago_with_tooltip(@project.mirror_last_update_at)}. - - if @project.mirror_ever_updated_successfully? - Last successful update #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}. - .panel-body - %pre - :preserve - #{@project.import_error.try(:strip)} +%p.light + When the repository has a remote mirror associated, it means that the remote repository configured in the Push to a remote repository section will automatically receive updates from the current repository. + +%hr.clearfix = form_for @project, url: namespace_project_mirror_path(@project.namespace, @project), html: { class: 'form-horizontal' } do |f| - if @project.errors.any? @@ -29,6 +19,25 @@ - @project.errors.full_messages.each do |msg| %p= msg + %h4 Pull from a remote repository + %p.light + Set up your project to automatically have its branches, tags, and commits updated from an upstream repository every hour. + = render "shared/mirror_update_button" + - if @project.mirror_last_update_success? + %span  Successfully updated #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}. + %hr.clearfix + + - if @project.mirror_last_update_failed? + .panel.panel-danger + .panel-heading + The repository failed to update #{time_ago_with_tooltip(@project.mirror_last_update_at)}. + - if @project.mirror_ever_updated_successfully? + Last successful update #{time_ago_with_tooltip(@project.mirror_last_successful_update_at)}. + .panel-body + %pre + :preserve + #{@project.import_error.try(:strip)} + .form-group .col-sm-offset-2.col-sm-10 .checkbox @@ -69,5 +78,46 @@ - if @project.builds_enabled? = render 'shared/mirror_trigger_builds_setting', f: f + %h4 Push to a remote repository + %p.light + Set up the remote repository that you want to update with the content of the current repository every hour. + = render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror + - if @remote_mirror.last_successful_update_at + %span  Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}. + + %hr.clearfix + + - if @remote_mirror.last_error.present? + .panel.panel-danger + .panel-heading + The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}. + - if @remote_mirror.last_successful_update_at + Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}. + .panel-body + %pre + :preserve + #{@remote_mirror.last_error.strip} + + = f.fields_for :remote_mirrors, @remote_mirror do |rm_form| + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = rm_form.label :enabled do + = rm_form.check_box :enabled + %strong + Remote Mirror Repository + .help-block + Automatically update the remote mirror's branches, tags, and commits from this repository every hour. + + .form-group.has-feedback + = rm_form.label :url, class: 'control-label' do + %span Git repository URL + .col-sm-10 + = rm_form.text_field :url, class: 'form-control', placeholder: 'https://username:password@gitlab.company.com/group/project.git', data: { 'safe-url' => @remote_mirror.url, 'full-url' => @remote_mirror.full_url } + - if @remote_mirror.has_credentials? + = link_to 'Show credentials', '#', class: 'toggle-remote-credentials' + .well.prepend-top-20 + The requirements for the URL format are the same mentioned in the Pull from a remote repository section. + .form-actions = f.submit "Save Changes", class: "btn btn-create" diff --git a/app/views/shared/_remote_mirror_update_button.html.haml b/app/views/shared/_remote_mirror_update_button.html.haml new file mode 100644 index 0000000000000000000000000000000000000000..d0772019d283f89e9b85cc251433bbb54d1413b0 --- /dev/null +++ b/app/views/shared/_remote_mirror_update_button.html.haml @@ -0,0 +1,10 @@ +- if @project.remote_mirror? + - if remote_mirror.update_in_progress? + %span.btn.disabled + = icon('refresh') + Updating… + - else + = link_to update_remote_now_namespace_project_mirror_path(@project.namespace, @project), method: :post, class: "btn" do + = icon('refresh') + Update Now + diff --git a/app/workers/repository_update_mirror_worker.rb b/app/workers/repository_update_mirror_worker.rb index d991a5bcde08db7b2806102fbe77d0f8e586edda..47a60f6a777c69c05f1047022f7b9339563a698a 100644 --- a/app/workers/repository_update_mirror_worker.rb +++ b/app/workers/repository_update_mirror_worker.rb @@ -12,8 +12,7 @@ def perform(project_id) result = Projects::UpdateMirrorService.new(@project, @current_user).execute if result[:status] == :error - project.update(import_error: result[:message]) - project.import_fail + project.mark_import_as_failed(result[:message]) return end diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..3341e08edafd5f27cd9ab2f61c41c4d5a14f48e1 --- /dev/null +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -0,0 +1,19 @@ +class RepositoryUpdateRemoteMirrorWorker + include Sidekiq::Worker + include Gitlab::ShellAdapter + + sidekiq_options queue: :gitlab_shell + + def perform(remote_mirror_id) + remote_mirror = RemoteMirror.find(remote_mirror_id) + project = remote_mirror.project + current_user = project.creator + result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror) + + if result[:status] == :error + remote_mirror.mark_as_failed(result[:message]) + else + remote_mirror.update_finish + end + end +end diff --git a/app/workers/update_all_mirrors_worker.rb b/app/workers/update_all_mirrors_worker.rb index e71163eee3e48e234c00f1aa1ab5735e9133171c..4d9c484bd8495176ab2b5e39ecc56c71bc038783 100644 --- a/app/workers/update_all_mirrors_worker.rb +++ b/app/workers/update_all_mirrors_worker.rb @@ -13,8 +13,7 @@ def fail_stuck_mirrors! where('mirror_last_update_at < ?', 1.day.ago) stuck.find_each(batch_size: 50) do |project| - project.import_fail - project.update_attribute(:import_error, 'The mirror update took too long to complete.') + project.mark_import_as_failed('The mirror update took too long to complete.') end end end diff --git a/app/workers/update_all_remote_mirrors_worker.rb b/app/workers/update_all_remote_mirrors_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..4849a5d92657135bb185a102e59ce911770eee7c --- /dev/null +++ b/app/workers/update_all_remote_mirrors_worker.rb @@ -0,0 +1,18 @@ +class UpdateAllRemoteMirrorsWorker + include Sidekiq::Worker + + def perform + fail_stuck_mirrors! + + RemoteMirror.find_each(batch_size: 50).each(&:sync) + end + + def fail_stuck_mirrors! + stuck = RemoteMirror.started. + where("last_update_at < ?", 1.day.ago) + + stuck.find_each(batch_size: 50) do |remote_mirror| + remote_mirror.mark_as_failed('The mirror update took too long to complete.') + end + end +end diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 9d840f15f719027eaa86cfd1d5a6bf25de59c1b4..3427e9267b1937cf578add8d8618f5740df5382f 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -327,6 +327,9 @@ def host(url) Settings.cron_jobs['update_all_mirrors_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['update_all_mirrors_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['update_all_mirrors_worker']['job_class'] = 'UpdateAllMirrorsWorker' +Settings.cron_jobs['update_all_remote_mirrors_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['update_all_remote_mirrors_worker']['cron'] ||= '30 * * * *' +Settings.cron_jobs['update_all_remote_mirrors_worker']['job_class'] = 'UpdateAllRemoteMirrorsWorker' Settings.cron_jobs['ldap_sync_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['ldap_sync_worker']['cron'] ||= '30 1 * * *' Settings.cron_jobs['ldap_sync_worker']['job_class'] = 'LdapSyncWorker' diff --git a/config/routes.rb b/config/routes.rb index def13f13b3ff130223c257141fb9903cdb336a9a..635bb1f8445ad883b06b9f4ab7b3abffe4794d09 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -698,6 +698,7 @@ resource :mirror, only: [:show, :update] do member do post :update_now + post :update_remote_now end end resources :git_hooks, constraints: { id: /\d+/ } diff --git a/db/migrate/20160321161032_create_remote_mirrors.rb b/db/migrate/20160321161032_create_remote_mirrors.rb new file mode 100644 index 0000000000000000000000000000000000000000..bbf2628d0534119b6e967f1b31f2ae3031734c68 --- /dev/null +++ b/db/migrate/20160321161032_create_remote_mirrors.rb @@ -0,0 +1,15 @@ +class CreateRemoteMirrors < ActiveRecord::Migration + def change + create_table :remote_mirrors do |t| + t.references :project, index: true, foreign_key: true + t.string :url + t.boolean :enabled, default: true + t.string :update_status + t.datetime :last_update_at + t.datetime :last_successful_update_at + t.string :last_error + + t.timestamps null: false + end + end +end diff --git a/db/migrate/20160330031024_add_mirror_credentials_to_remote_mirrors.rb b/db/migrate/20160330031024_add_mirror_credentials_to_remote_mirrors.rb new file mode 100644 index 0000000000000000000000000000000000000000..dcd387e19498ef700fa7ad68b3c6e43741c64671 --- /dev/null +++ b/db/migrate/20160330031024_add_mirror_credentials_to_remote_mirrors.rb @@ -0,0 +1,7 @@ +class AddMirrorCredentialsToRemoteMirrors < ActiveRecord::Migration + def change + add_column :remote_mirrors, :encrypted_credentials, :text + add_column :remote_mirrors, :encrypted_credentials_iv, :text + add_column :remote_mirrors, :encrypted_credentials_salt, :text + end +end diff --git a/db/schema.rb b/db/schema.rb index 5ef6c4d079d6d2c3522b90224a65aed37b4bb674..758632d5791233413d6be7115d58071b9fcc2435 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20160320204112) do +ActiveRecord::Schema.define(version: 20160330031024) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -883,6 +883,23 @@ add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree add_index "releases", ["project_id"], name: "index_releases_on_project_id", using: :btree + create_table "remote_mirrors", force: :cascade do |t| + t.integer "project_id" + t.string "url" + t.boolean "enabled", default: true + t.string "update_status" + t.datetime "last_update_at" + t.datetime "last_successful_update_at" + t.string "last_error" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "encrypted_credentials" + t.text "encrypted_credentials_iv" + t.text "encrypted_credentials_salt" + end + + add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree + create_table "sent_notifications", force: :cascade do |t| t.integer "project_id" t.integer "noteable_id" diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index b09c37f1ca252c4c6c7bde520eb28a8bedacfadd..f09c83603fa08056722cccde1b66e9444dfa9d6d 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -271,6 +271,38 @@ def exists?(dir_name) File.exists?(full_path(dir_name)) end + # Push branch to remote repository + # + # project_name - project's name with namespace + # remote_name - remote name + # branch_name - remote branch name + # + # Ex. + # push_branches('upstream', 'feature') + # + def push_branches(project_name, remote_name, branch_names) + args = [gitlab_shell_projects_path, 'push-branches', "#{project_name}.git", remote_name, *branch_names] + output, status = Popen::popen(args) + raise Error, output unless status.zero? + true + end + + # Delete branch from remote repository + # + # project_name - project's name with namespace + # remote_name - remote name + # branch_name - remote branch name + # + # Ex. + # delete_remote_branches('upstream', 'feature') + # + def delete_remote_branches(project_name, remote_name, branch_names) + args = [gitlab_shell_projects_path, 'delete-remote-branches', "#{project_name}.git", remote_name, *branch_names] + output, status = Popen::popen(args) + raise Error, output unless status.zero? + true + end + protected def gitlab_shell_path diff --git a/lib/gitlab/import_url.rb b/lib/gitlab/import_url.rb new file mode 100644 index 0000000000000000000000000000000000000000..953ad854045586cf6a634832c5e71cf18ad921d0 --- /dev/null +++ b/lib/gitlab/import_url.rb @@ -0,0 +1,43 @@ +# I'm borrowing this class from: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3066 +# So we should fix the conflict once the CE -> EE merge process starts. +module Gitlab + class ImportUrl + def initialize(url, credentials: nil) + @url = URI.parse(url) + @credentials = credentials + end + + def sanitized_url + @sanitized_url ||= safe_url.to_s + end + + def credentials + @credentials ||= { user: @url.user, password: @url.password } + end + + def full_url + @full_url ||= generate_full_url.to_s + end + + private + + def generate_full_url + return @url unless valid_credentials? + @full_url = @url.dup + @full_url.user = credentials[:user] + @full_url.password = credentials[:password] + @full_url + end + + def safe_url + safe_url = @url.dup + safe_url.password = nil + safe_url.user = nil + safe_url + end + + def valid_credentials? + credentials && credentials.is_a?(Hash) && credentials.any? + end + end +end diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index 66b38d85652d9a805922e53e0ae0af20fdd18124..3d129857b3771b7da5ced3968b526c2fa7f34667 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -2,40 +2,54 @@ # # Table name: projects # -# id :integer not null, primary key -# name :string(255) -# path :string(255) -# description :text -# created_at :datetime -# updated_at :datetime -# creator_id :integer -# issues_enabled :boolean default(TRUE), not null -# wall_enabled :boolean default(TRUE), not null -# merge_requests_enabled :boolean default(TRUE), not null -# wiki_enabled :boolean default(TRUE), not null -# namespace_id :integer -# issues_tracker :string(255) default("gitlab"), not null -# issues_tracker_id :string(255) -# snippets_enabled :boolean default(TRUE), not null -# last_activity_at :datetime -# import_url :string(255) -# visibility_level :integer default(0), not null -# archived :boolean default(FALSE), not null -# avatar :string(255) -# import_status :string(255) -# repository_size :float default(0.0) -# star_count :integer default(0), not null -# import_type :string(255) -# import_source :string(255) -# commit_count :integer default(0) -# import_error :text -# ci_id :integer -# builds_enabled :boolean default(TRUE), not null -# shared_runners_enabled :boolean default(TRUE), not null -# runners_token :string -# build_coverage_regex :string -# build_allow_git_fetch :boolean default(TRUE), not null -# build_timeout :integer default(3600), not null +# id :integer not null, primary key +# name :string(255) +# path :string(255) +# description :text +# created_at :datetime +# updated_at :datetime +# creator_id :integer +# issues_enabled :boolean default(TRUE), not null +# wall_enabled :boolean default(TRUE), not null +# merge_requests_enabled :boolean default(TRUE), not null +# wiki_enabled :boolean default(TRUE), not null +# namespace_id :integer +# issues_tracker :string(255) default("gitlab"), not null +# issues_tracker_id :string(255) +# snippets_enabled :boolean default(TRUE), not null +# last_activity_at :datetime +# import_url :string(255) +# visibility_level :integer default(0), not null +# archived :boolean default(FALSE), not null +# avatar :string(255) +# import_status :string(255) +# repository_size :float default(0.0) +# star_count :integer default(0), not null +# import_type :string(255) +# import_source :string(255) +# commit_count :integer default(0) +# import_error :text +# ci_id :integer +# builds_enabled :boolean default(TRUE), not null +# shared_runners_enabled :boolean default(TRUE), not null +# runners_token :string +# build_coverage_regex :string +# build_allow_git_fetch :boolean default(TRUE), not null +# build_timeout :integer default(3600), not null +# pending_delete :boolean default(FALSE) +# public_builds :boolean default(TRUE), not null +# merge_requests_template :text +# merge_requests_rebase_enabled :boolean default(FALSE) +# approvals_before_merge :integer default(0), not null +# reset_approvals_on_push :boolean default(TRUE) +# merge_requests_ff_only_enabled :boolean default(FALSE) +# issues_template :text +# mirror :boolean default(FALSE), not null +# mirror_last_update_at :datetime +# mirror_last_successful_update_at :datetime +# mirror_user_id :integer +# mirror_trigger_builds :boolean default(FALSE), not null +# main_language :string # FactoryGirl.define do diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index b8baf86208a0063ab6d2334c06e3f64c2d81d4e3..bcdc5c41e73ff04d224eb1627b817ae22046affb 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2,40 +2,54 @@ # # Table name: projects # -# id :integer not null, primary key -# name :string(255) -# path :string(255) -# description :text -# created_at :datetime -# updated_at :datetime -# creator_id :integer -# issues_enabled :boolean default(TRUE), not null -# wall_enabled :boolean default(TRUE), not null -# merge_requests_enabled :boolean default(TRUE), not null -# wiki_enabled :boolean default(TRUE), not null -# namespace_id :integer -# issues_tracker :string(255) default("gitlab"), not null -# issues_tracker_id :string(255) -# snippets_enabled :boolean default(TRUE), not null -# last_activity_at :datetime -# import_url :string(255) -# visibility_level :integer default(0), not null -# archived :boolean default(FALSE), not null -# avatar :string(255) -# import_status :string(255) -# repository_size :float default(0.0) -# star_count :integer default(0), not null -# import_type :string(255) -# import_source :string(255) -# commit_count :integer default(0) -# import_error :text -# ci_id :integer -# builds_enabled :boolean default(TRUE), not null -# shared_runners_enabled :boolean default(TRUE), not null -# runners_token :string -# build_coverage_regex :string -# build_allow_git_fetch :boolean default(TRUE), not null -# build_timeout :integer default(3600), not null +# id :integer not null, primary key +# name :string(255) +# path :string(255) +# description :text +# created_at :datetime +# updated_at :datetime +# creator_id :integer +# issues_enabled :boolean default(TRUE), not null +# wall_enabled :boolean default(TRUE), not null +# merge_requests_enabled :boolean default(TRUE), not null +# wiki_enabled :boolean default(TRUE), not null +# namespace_id :integer +# issues_tracker :string(255) default("gitlab"), not null +# issues_tracker_id :string(255) +# snippets_enabled :boolean default(TRUE), not null +# last_activity_at :datetime +# import_url :string(255) +# visibility_level :integer default(0), not null +# archived :boolean default(FALSE), not null +# avatar :string(255) +# import_status :string(255) +# repository_size :float default(0.0) +# star_count :integer default(0), not null +# import_type :string(255) +# import_source :string(255) +# commit_count :integer default(0) +# import_error :text +# ci_id :integer +# builds_enabled :boolean default(TRUE), not null +# shared_runners_enabled :boolean default(TRUE), not null +# runners_token :string +# build_coverage_regex :string +# build_allow_git_fetch :boolean default(TRUE), not null +# build_timeout :integer default(3600), not null +# pending_delete :boolean default(FALSE) +# public_builds :boolean default(TRUE), not null +# merge_requests_template :text +# merge_requests_rebase_enabled :boolean default(FALSE) +# approvals_before_merge :integer default(0), not null +# reset_approvals_on_push :boolean default(TRUE) +# merge_requests_ff_only_enabled :boolean default(FALSE) +# issues_template :text +# mirror :boolean default(FALSE), not null +# mirror_last_update_at :datetime +# mirror_last_successful_update_at :datetime +# mirror_user_id :integer +# mirror_trigger_builds :boolean default(FALSE), not null +# main_language :string # require 'spec_helper' diff --git a/spec/services/projects/update_remote_mirror_service_spec.rb b/spec/services/projects/update_remote_mirror_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..d0b31cf32019127a9836e9344195958b91914c71 --- /dev/null +++ b/spec/services/projects/update_remote_mirror_service_spec.rb @@ -0,0 +1,117 @@ +require 'spec_helper' + +describe Projects::UpdateRemoteMirrorService do + let(:project) { create(:project) } + let(:remote_project) { create(:forked_project_with_submodules) } + let(:repository) { project.repository } + let(:remote_repository) { remote_project.repository } + let(:remote_mirror) { project.remote_mirrors.create!(url: remote_project.http_url_to_repo) } + let(:all_branches) { ["master", 'existing-branch', "'test'", "empty-branch", "feature", "feature_conflict", "fix", "flatten-dir", "improve/awesome", "lfs", "markdown"] } + + subject { described_class.new(project, project.creator) } + + describe "#execute" do + before do + create_branch(repository, 'existing-branch') + end + + it "fetches the remote repository" do + expect(repository).to receive(:fetch_remote).with(remote_mirror.ref_name) + + subject.execute(remote_mirror) + end + + it "succeeds" do + allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.ref_name, all_branches) } + + result = subject.execute(remote_mirror) + + expect(result[:status]).to eq(:success) + end + + describe 'Updating branches' do + it "push all the branches the first time" do + allow(repository).to receive(:fetch_remote) + + expect(repository).to receive(:push_branches).with(project.path_with_namespace, remote_mirror.ref_name, all_branches) + + subject.execute(remote_mirror) + end + + it "does not push anything is remote is up to date" do + allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.ref_name, all_branches) } + + expect(repository).not_to receive(:push_branches) + + subject.execute(remote_mirror) + end + + it "sync new branches" do + allow(repository).to receive(:fetch_remote) { sync_remote(repository, remote_mirror.ref_name, all_branches) } + create_branch(repository, 'my-new-branch') + + expect(repository).to receive(:push_branches).with(project.path_with_namespace, remote_mirror.ref_name, ['my-new-branch']) + + subject.execute(remote_mirror) + end + + it "sync updated branches" do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.ref_name, all_branches) + update_branch(repository, 'existing-branch') + end + + expect(repository).to receive(:push_branches).with(project.path_with_namespace, remote_mirror.ref_name, ['existing-branch']) + + subject.execute(remote_mirror) + end + + it "sync deleted branches" do + allow(repository).to receive(:fetch_remote) do + sync_remote(repository, remote_mirror.ref_name, all_branches) + delete_branch(repository, 'existing-branch') + end + + expect(repository).to receive(:delete_remote_branches).with(project.path_with_namespace, remote_mirror.ref_name, ['existing-branch']) + + subject.execute(remote_mirror) + end + end + + end + + def create_branch(repository, branch_name) + rugged = repository.rugged + masterrev = repository.find_branch('master').target + parentrev = repository.commit(masterrev).parent_id + + rugged.references.create("refs/heads/#{branch_name}", parentrev) + + repository.expire_branches_cache + end + + def sync_remote(repository, remote_name, all_branches) + rugged = repository.rugged + + all_branches.each do |branch| + target = repository.find_branch(branch).try(:target) + rugged.references.create("refs/remotes/#{remote_name}/#{branch}", target) if target + end + end + + def update_branch(repository, branch) + rugged = repository.rugged + masterrev = repository.find_branch('master').target + + # # Updated existing branch + rugged.references.create("refs/heads/#{branch}", masterrev, force: true) + repository.expire_branches_cache + end + + def delete_branch(repository, branch) + rugged = repository.rugged + + rugged.references.delete("refs/heads/#{branch}") + repository.expire_branches_cache + end +end