diff --git a/app/assets/javascripts/build.js b/app/assets/javascripts/build.js index fe54ecffdfe8446e9b879bc32ea72d4a60a7ad94..0aad95c2fe39eef4b1a34e2f04ab3b20ac4485cf 100644 --- a/app/assets/javascripts/build.js +++ b/app/assets/javascripts/build.js @@ -1,24 +1,28 @@ -/* eslint-disable func-names, space-before-function-paren, no-var, prefer-rest-params, wrap-iife, no-use-before-define, no-param-reassign, quotes, yoda, no-else-return, consistent-return, comma-dangle, object-shorthand, prefer-template, one-var, one-var-declaration-per-line, no-unused-vars, max-len, vars-on-top */ +/* eslint-disable func-names, wrap-iife, no-use-before-define, +consistent-return, prefer-rest-params */ /* global Breakpoints */ -var bind = function(fn, me) { return function() { return fn.apply(me, arguments); }; }; -var AUTO_SCROLL_OFFSET = 75; -var DOWN_BUILD_TRACE = '#down-build-trace'; +const bind = function (fn, me) { return function () { return fn.apply(me, arguments); }; }; +const AUTO_SCROLL_OFFSET = 75; +const DOWN_BUILD_TRACE = '#down-build-trace'; -window.Build = (function() { +window.Build = (function () { Build.timeout = null; Build.state = null; function Build(options) { - options = options || $('.js-build-options').data(); - this.pageUrl = options.pageUrl; - this.buildUrl = options.buildUrl; - this.buildStatus = options.buildStatus; - this.state = options.logState; - this.buildStage = options.buildStage; - this.updateDropdown = bind(this.updateDropdown, this); + this.options = options || $('.js-build-options').data(); + + this.pageUrl = this.options.pageUrl; + this.buildUrl = this.options.buildUrl; + this.buildStatus = this.options.buildStatus; + this.state = this.options.logState; + this.buildStage = this.options.buildStage; this.$document = $(document); + + this.updateDropdown = bind(this.updateDropdown, this); + this.$body = $('body'); this.$buildTrace = $('#build-trace'); this.$autoScrollContainer = $('.autoscroll-container'); @@ -29,112 +33,112 @@ window.Build = (function() { this.$scrollTopBtn = $('#scroll-top'); this.$scrollBottomBtn = $('#scroll-bottom'); this.$buildRefreshAnimation = $('.js-build-refresh'); + this.$buildScroll = $('#js-build-scroll'); + this.$truncatedInfo = $('.js-truncated-info'); clearTimeout(Build.timeout); // Init breakpoint checker this.bp = Breakpoints.get(); this.initSidebar(); - this.$buildScroll = $('#js-build-scroll'); - this.populateJobs(this.buildStage); this.updateStageDropdownText(this.buildStage); this.sidebarOnResize(); - this.$document.off('click', '.js-sidebar-build-toggle').on('click', '.js-sidebar-build-toggle', this.sidebarOnClick.bind(this)); - this.$document.off('click', '.stage-item').on('click', '.stage-item', this.updateDropdown); + this.$document + .off('click', '.js-sidebar-build-toggle') + .on('click', '.js-sidebar-build-toggle', this.sidebarOnClick.bind(this)); + + this.$document + .off('click', '.stage-item') + .on('click', '.stage-item', this.updateDropdown); + this.$document.on('scroll', this.initScrollMonitor.bind(this)); - $(window).off('resize.build').on('resize.build', this.sidebarOnResize.bind(this)); - $('a', this.$buildScroll).off('click.stepTrace').on('click.stepTrace', this.stepTrace); + + $(window) + .off('resize.build') + .on('resize.build', this.sidebarOnResize.bind(this)); + + $('a', this.$buildScroll) + .off('click.stepTrace') + .on('click.stepTrace', this.stepTrace); + this.updateArtifactRemoveDate(); - if ($('#build-trace').length) { - this.getInitialBuildTrace(); - this.initScrollButtonAffix(); - } + this.initScrollButtonAffix(); this.invokeBuildTrace(); } - Build.prototype.initSidebar = function() { + Build.prototype.initSidebar = function () { this.$sidebar = $('.js-build-sidebar'); this.$sidebar.niceScroll(); - this.$document.off('click', '.js-sidebar-build-toggle').on('click', '.js-sidebar-build-toggle', this.toggleSidebar); - }; - - Build.prototype.location = function() { - return window.location.href.split("#")[0]; + this.$document + .off('click', '.js-sidebar-build-toggle') + .on('click', '.js-sidebar-build-toggle', this.toggleSidebar); }; - Build.prototype.invokeBuildTrace = function() { - var continueRefreshStatuses = ['running', 'pending']; - // Continue to update build trace when build is running or pending - if (continueRefreshStatuses.indexOf(this.buildStatus) !== -1) { - // Check for new build output if user still watching build page - // Only valid for runnig build when output changes during time - Build.timeout = setTimeout((function(_this) { - return function() { - if (_this.location() === _this.pageUrl) { - return _this.getBuildTrace(); - } - }; - })(this), 4000); - } + Build.prototype.invokeBuildTrace = function () { + return this.getBuildTrace(); }; - Build.prototype.getInitialBuildTrace = function() { - var removeRefreshStatuses = ['success', 'failed', 'canceled', 'skipped']; - + Build.prototype.getBuildTrace = function () { return $.ajax({ - url: this.buildUrl, + url: `${this.pageUrl}/trace.json`, dataType: 'json', - success: function(buildData) { - $('.js-build-output').html(buildData.trace_html); + data: { + state: this.state, + }, + success: ((log) => { + const $buildContainer = $('.js-build-output'); + gl.utils.setCiStatusFavicon(`${this.pageUrl}/status.json`); - if (window.location.hash === DOWN_BUILD_TRACE) { - $("html,body").scrollTop(this.$buildTrace.height()); + + if (log.state) { + this.state = log.state; } - if (removeRefreshStatuses.indexOf(buildData.status) !== -1) { + + if (log.append) { + $buildContainer.append(log.html); + } else { + $buildContainer.html(log.html); + if (log.truncated) { + $('.js-truncated-info-size').html(` ${log.size} `); + this.$truncatedInfo.removeClass('hidden'); + this.initAffixTruncatedInfo(); + } else { + this.$truncatedInfo.addClass('hidden'); + } + } + + this.checkAutoscroll(); + + if (!log.complete) { + Build.timeout = setTimeout(() => { + this.invokeBuildTrace(); + }, 4000); + } else { this.$buildRefreshAnimation.remove(); - return this.initScrollMonitor(); } - }.bind(this) - }); - }; - Build.prototype.getBuildTrace = function() { - return $.ajax({ - url: this.pageUrl + "/trace.json?state=" + (encodeURIComponent(this.state)), - dataType: "json", - success: (function(_this) { - return function(log) { - var pageUrl; - - if (log.state) { - _this.state = log.state; - } - _this.invokeBuildTrace(); - if (log.status === "running") { - if (log.append) { - $('.js-build-output').append(log.html); - } else { - $('.js-build-output').html(log.html); - } - return _this.checkAutoscroll(); - } else if (log.status !== _this.buildStatus) { - pageUrl = _this.pageUrl; - if (_this.$autoScrollStatus.data('state') === 'enabled') { - pageUrl += DOWN_BUILD_TRACE; - } - - return gl.utils.visitUrl(pageUrl); + if (log.status !== this.buildStatus) { + let pageUrl = this.pageUrl; + + if (this.$autoScrollStatus.data('state') === 'enabled') { + pageUrl += DOWN_BUILD_TRACE; } - }; - })(this) + + gl.utils.visitUrl(pageUrl); + } + }), + error: () => { + this.$buildRefreshAnimation.remove(); + return this.initScrollMonitor(); + }, }); }; - Build.prototype.checkAutoscroll = function() { - if (this.$autoScrollStatus.data("state") === "enabled") { - return $("html,body").scrollTop(this.$buildTrace.height()); + Build.prototype.checkAutoscroll = function () { + if (this.$autoScrollStatus.data('state') === 'enabled') { + return $('html,body').scrollTop(this.$buildTrace.height()); } // Handle a situation where user started new build @@ -146,7 +150,7 @@ window.Build = (function() { } }; - Build.prototype.initScrollButtonAffix = function() { + Build.prototype.initScrollButtonAffix = function () { // Hide everything initially this.$scrollTopBtn.hide(); this.$scrollBottomBtn.hide(); @@ -167,15 +171,17 @@ window.Build = (function() { // - Show Top Arrow button // - Show Bottom Arrow button // - Disable Autoscroll and hide indicator (when build is running) - Build.prototype.initScrollMonitor = function() { - if (!gl.utils.isInViewport(this.$upBuildTrace.get(0)) && !gl.utils.isInViewport(this.$downBuildTrace.get(0))) { + Build.prototype.initScrollMonitor = function () { + if (!gl.utils.isInViewport(this.$upBuildTrace.get(0)) && + !gl.utils.isInViewport(this.$downBuildTrace.get(0))) { // User is somewhere in middle of Build Log this.$scrollTopBtn.show(); if (this.buildStatus === 'success' || this.buildStatus === 'failed') { // Check if Build is completed this.$scrollBottomBtn.show(); - } else if (this.$buildRefreshAnimation.is(':visible') && !gl.utils.isInViewport(this.$buildRefreshAnimation.get(0))) { + } else if (this.$buildRefreshAnimation.is(':visible') && + !gl.utils.isInViewport(this.$buildRefreshAnimation.get(0))) { this.$scrollBottomBtn.show(); } else { this.$scrollBottomBtn.hide(); @@ -186,10 +192,13 @@ window.Build = (function() { this.$autoScrollContainer.hide(); this.$autoScrollStatusText.removeClass('animate'); } else { - this.$autoScrollContainer.css({ top: this.$body.outerHeight() - AUTO_SCROLL_OFFSET }).show(); + this.$autoScrollContainer.css({ + top: this.$body.outerHeight() - AUTO_SCROLL_OFFSET, + }).show(); this.$autoScrollStatusText.addClass('animate'); } - } else if (gl.utils.isInViewport(this.$upBuildTrace.get(0)) && !gl.utils.isInViewport(this.$downBuildTrace.get(0))) { + } else if (gl.utils.isInViewport(this.$upBuildTrace.get(0)) && + !gl.utils.isInViewport(this.$downBuildTrace.get(0))) { // User is at Top of Build Log this.$scrollTopBtn.hide(); @@ -197,17 +206,22 @@ window.Build = (function() { this.$autoScrollContainer.hide(); this.$autoScrollStatusText.removeClass('animate'); - } else if ((!gl.utils.isInViewport(this.$upBuildTrace.get(0)) && gl.utils.isInViewport(this.$downBuildTrace.get(0))) || - (this.$buildRefreshAnimation.is(':visible') && gl.utils.isInViewport(this.$buildRefreshAnimation.get(0)))) { + } else if ((!gl.utils.isInViewport(this.$upBuildTrace.get(0)) && + gl.utils.isInViewport(this.$downBuildTrace.get(0))) || + (this.$buildRefreshAnimation.is(':visible') && + gl.utils.isInViewport(this.$buildRefreshAnimation.get(0)))) { // User is at Bottom of Build Log this.$scrollTopBtn.show(); this.$scrollBottomBtn.hide(); // Show and Reposition Autoscroll Status Indicator - this.$autoScrollContainer.css({ top: this.$body.outerHeight() - AUTO_SCROLL_OFFSET }).show(); + this.$autoScrollContainer.css({ + top: this.$body.outerHeight() - AUTO_SCROLL_OFFSET, + }).show(); this.$autoScrollStatusText.addClass('animate'); - } else if (gl.utils.isInViewport(this.$upBuildTrace.get(0)) && gl.utils.isInViewport(this.$downBuildTrace.get(0))) { + } else if (gl.utils.isInViewport(this.$upBuildTrace.get(0)) && + gl.utils.isInViewport(this.$downBuildTrace.get(0))) { // Build Log height is small this.$scrollTopBtn.hide(); @@ -218,65 +232,81 @@ window.Build = (function() { this.$autoScrollStatusText.removeClass('animate'); } - if (this.buildStatus === "running" || this.buildStatus === "pending") { + if (this.buildStatus === 'running' || this.buildStatus === 'pending') { // Check if Refresh Animation is in Viewport and enable Autoscroll, disable otherwise. - this.$autoScrollStatus.data("state", gl.utils.isInViewport(this.$buildRefreshAnimation.get(0)) ? 'enabled' : 'disabled'); + this.$autoScrollStatus.data( + 'state', + gl.utils.isInViewport(this.$buildRefreshAnimation.get(0)) ? 'enabled' : 'disabled', + ); } }; - Build.prototype.shouldHideSidebarForViewport = function() { - var bootstrapBreakpoint; - bootstrapBreakpoint = this.bp.getBreakpointSize(); + Build.prototype.shouldHideSidebarForViewport = function () { + const bootstrapBreakpoint = this.bp.getBreakpointSize(); return bootstrapBreakpoint === 'xs' || bootstrapBreakpoint === 'sm'; }; - Build.prototype.toggleSidebar = function(shouldHide) { - var shouldShow = typeof shouldHide === 'boolean' ? !shouldHide : undefined; + Build.prototype.toggleSidebar = function (shouldHide) { + const shouldShow = typeof shouldHide === 'boolean' ? !shouldHide : undefined; + this.$buildScroll.toggleClass('sidebar-expanded', shouldShow) .toggleClass('sidebar-collapsed', shouldHide); + this.$truncatedInfo.toggleClass('sidebar-expanded', shouldShow) + .toggleClass('sidebar-collapsed', shouldHide); this.$sidebar.toggleClass('right-sidebar-expanded', shouldShow) .toggleClass('right-sidebar-collapsed', shouldHide); }; - Build.prototype.sidebarOnResize = function() { + Build.prototype.sidebarOnResize = function () { this.toggleSidebar(this.shouldHideSidebarForViewport()); }; - Build.prototype.sidebarOnClick = function() { + Build.prototype.sidebarOnClick = function () { if (this.shouldHideSidebarForViewport()) this.toggleSidebar(); }; - Build.prototype.updateArtifactRemoveDate = function() { - var $date, date; - $date = $('.js-artifacts-remove'); + Build.prototype.updateArtifactRemoveDate = function () { + const $date = $('.js-artifacts-remove'); if ($date.length) { - date = $date.text(); - return $date.text(gl.utils.timeFor(new Date(date.replace(/([0-9]+)-([0-9]+)-([0-9]+)/g, '$1/$2/$3')), ' ')); + const date = $date.text(); + return $date.text( + gl.utils.timeFor(new Date(date.replace(/([0-9]+)-([0-9]+)-([0-9]+)/g, '$1/$2/$3')), ' '), + ); } }; - Build.prototype.populateJobs = function(stage) { + Build.prototype.populateJobs = function (stage) { $('.build-job').hide(); - $('.build-job[data-stage="' + stage + '"]').show(); + $(`.build-job[data-stage="${stage}"]`).show(); }; - Build.prototype.updateStageDropdownText = function(stage) { + Build.prototype.updateStageDropdownText = function (stage) { $('.stage-selection').text(stage); }; - Build.prototype.updateDropdown = function(e) { + Build.prototype.updateDropdown = function (e) { e.preventDefault(); - var stage = e.currentTarget.text; + const stage = e.currentTarget.text; this.updateStageDropdownText(stage); this.populateJobs(stage); }; - Build.prototype.stepTrace = function(e) { - var $currentTarget; + Build.prototype.stepTrace = function (e) { e.preventDefault(); - $currentTarget = $(e.currentTarget); + + const $currentTarget = $(e.currentTarget); $.scrollTo($currentTarget.attr('href'), { - offset: 0 + offset: 0, + }); + }; + + Build.prototype.initAffixTruncatedInfo = function () { + const offsetTop = this.$buildTrace.offset().top; + + this.$truncatedInfo.affix({ + offset: { + top: offsetTop, + }, }); }; diff --git a/app/assets/stylesheets/pages/builds.scss b/app/assets/stylesheets/pages/builds.scss index 969fc75c6eb220a5939484f6622bf074c7509150..03fddaeb163f61ff2c95f6db6669647a4b3bd27f 100644 --- a/app/assets/stylesheets/pages/builds.scss +++ b/app/assets/stylesheets/pages/builds.scss @@ -57,6 +57,37 @@ margin-right: 5px; } } + + .truncated-info { + text-align: center; + border-bottom: 1px solid; + background-color: $black-transparent; + height: 45px; + + &.affix { + top: 0; + } + + // with sidebar + &.affix.sidebar-expanded { + right: 312px; + left: 22px; + } + + // without sidebar + &.affix.sidebar-collapsed { + right: 20px; + left: 20px; + } + + &.affix-top { + position: absolute; + top: 0; + margin: 0 auto; + right: 5px; + left: 5px; + } + } } .scroll-controls { @@ -186,6 +217,7 @@ white-space: pre; overflow-x: auto; font-size: 12px; + position: relative; .fa-refresh { font-size: 24px; diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb index 3f3c90a49ab8f3eca717b3a988a135cf81e42016..add66ce9f84767fe46646c1309e709c3d0df058e 100644 --- a/app/controllers/projects/builds_controller.rb +++ b/app/controllers/projects/builds_controller.rb @@ -31,25 +31,25 @@ def show @builds = @project.pipelines.find_by_sha(@build.sha).builds.order('id DESC') @builds = @builds.where("id not in (?)", @build.id) @pipeline = @build.pipeline - - respond_to do |format| - format.html - format.json do - render json: { - id: @build.id, - status: @build.status, - trace_html: @build.trace_html - } - end - end end def trace - respond_to do |format| - format.json do - state = params[:state].presence - render json: @build.trace_with_state(state: state). - merge!(id: @build.id, status: @build.status) + build.trace.read do |stream| + respond_to do |format| + format.json do + result = { + id: @build.id, status: @build.status, complete: @build.complete? + } + + if stream.valid? + stream.limit + state = params[:state].presence + trace = stream.html_with_state(state) + result.merge!(trace.to_h) + end + + render json: result + end end end end @@ -86,10 +86,12 @@ def erase end def raw - if @build.has_trace_file? - send_file @build.trace_file_path, type: 'text/plain; charset=utf-8', disposition: 'inline' - else - render_404 + build.trace.read do |stream| + if stream.file? + send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline' + else + render_404 + end end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index bfebcd43b7282de8c09904516a8ef8a129d0caa4..1d63c930aad4c786a3c5461d8d7cb7869eb65e07 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -172,19 +172,6 @@ def depends_on_builds latest_builds.where('stage_idx < ?', stage_idx) end - def trace_html(**args) - trace_with_state(**args)[:html] || '' - end - - def trace_with_state(state: nil, last_lines: nil) - trace_ansi = trace(last_lines: last_lines) - if trace_ansi.present? - Ci::Ansi2html.convert(trace_ansi, state) - else - {} - end - end - def timeout project.build_timeout end @@ -245,136 +232,35 @@ def allow_git_fetch end def update_coverage - coverage = extract_coverage(trace, coverage_regex) + coverage = trace.extract_coverage(coverage_regex) update_attributes(coverage: coverage) if coverage.present? end - def extract_coverage(text, regex) - return unless regex - - matches = text.scan(Regexp.new(regex)).last - matches = matches.last if matches.is_a?(Array) - coverage = matches.gsub(/\d+(\.\d+)?/).first - - if coverage.present? - coverage.to_f - end - rescue - # if bad regex or something goes wrong we dont want to interrupt transition - # so we just silentrly ignore error for now - end - - def has_trace_file? - File.exist?(path_to_trace) || has_old_trace_file? + def trace + Gitlab::Ci::Trace.new(self) end def has_trace? - raw_trace.present? + trace.exist? end - def raw_trace(last_lines: nil) - if File.exist?(trace_file_path) - Gitlab::Ci::TraceReader.new(trace_file_path). - read(last_lines: last_lines) - else - # backward compatibility - read_attribute :trace - end + def trace=(data) + raise NotImplementedError end - ## - # Deprecated - # - # This is a hotfix for CI build data integrity, see #4246 - def has_old_trace_file? - project.ci_id && File.exist?(old_path_to_trace) - end - - def trace(last_lines: nil) - hide_secrets(raw_trace(last_lines: last_lines)) + def old_trace + read_attribute(:trace) end - def trace_length - if raw_trace - raw_trace.bytesize - else - 0 - end - end - - def trace=(trace) - recreate_trace_dir - trace = hide_secrets(trace) - File.write(path_to_trace, trace) - end - - def recreate_trace_dir - unless Dir.exist?(dir_to_trace) - FileUtils.mkdir_p(dir_to_trace) - end - end - private :recreate_trace_dir - - def append_trace(trace_part, offset) - recreate_trace_dir - touch if needs_touch? - - trace_part = hide_secrets(trace_part) - - File.truncate(path_to_trace, offset) if File.exist?(path_to_trace) - File.open(path_to_trace, 'ab') do |f| - f.write(trace_part) - end + def erase_old_trace! + write_attribute(:trace, nil) + save end def needs_touch? Time.now - updated_at > 15.minutes.to_i end - def trace_file_path - if has_old_trace_file? - old_path_to_trace - else - path_to_trace - end - end - - def dir_to_trace - File.join( - Settings.gitlab_ci.builds_path, - created_at.utc.strftime("%Y_%m"), - project.id.to_s - ) - end - - def path_to_trace - "#{dir_to_trace}/#{id}.log" - end - - ## - # Deprecated - # - # This is a hotfix for CI build data integrity, see #4246 - # Should be removed in 8.4, after CI files migration has been done. - # - def old_dir_to_trace - File.join( - Settings.gitlab_ci.builds_path, - created_at.utc.strftime("%Y_%m"), - project.ci_id.to_s - ) - end - - ## - # Deprecated - # - # This is a hotfix for CI build data integrity, see #4246 - # Should be removed in 8.4, after CI files migration has been done. - # - def old_path_to_trace - "#{old_dir_to_trace}/#{id}.log" - end - ## # Deprecated # @@ -556,6 +442,15 @@ def empty_dependencies? options[:dependencies]&.empty? end + def hide_secrets(trace) + return unless trace + + trace = trace.dup + Ci::MaskSecret.mask!(trace, project.runners_token) if project + Ci::MaskSecret.mask!(trace, token) + trace + end + private def update_artifacts_size @@ -567,7 +462,7 @@ def update_artifacts_size end def erase_trace! - self.trace = nil + trace.erase! end def update_erased!(user = nil) @@ -629,15 +524,6 @@ def build_attributes_from_config pipeline.config_processor.build_attributes(name) end - def hide_secrets(trace) - return unless trace - - trace = trace.dup - Ci::MaskSecret.mask!(trace, project.runners_token) if project - Ci::MaskSecret.mask!(trace, token) - trace - end - def update_project_statistics return unless project diff --git a/app/views/notify/pipeline_failed_email.html.haml b/app/views/notify/pipeline_failed_email.html.haml index 4beb6fcee5d9d6e6325852c79592289d37048a67..a83faa839df44b7355591dd8ab0d46ed816b38f6 100644 --- a/app/views/notify/pipeline_failed_email.html.haml +++ b/app/views/notify/pipeline_failed_email.html.haml @@ -137,6 +137,6 @@ - if build.has_trace? %td{ colspan: "2", style: "font-family:'Helvetica Neue',Helvetica,Arial,sans-serif;padding:0 0 15px;" } %pre{ style: "font-family:Monaco,'Lucida Console','Courier New',Courier,monospace;background-color:#fafafa;border-radius:3px;overflow:hidden;white-space:pre-wrap;word-break:break-all;font-size:13px;line-height:1.4;padding:12px;color:#333333;margin:0;" } - = build.trace_html(last_lines: 10).html_safe + = build.trace.html(last_lines: 10).html_safe - else %td{ colspan: "2" } diff --git a/app/views/notify/pipeline_failed_email.text.erb b/app/views/notify/pipeline_failed_email.text.erb index c1a4ea40cf5b2080dc273f02f0358de93daee454..294238eee5135c19e08e5078d9498639bc09cfe1 100644 --- a/app/views/notify/pipeline_failed_email.text.erb +++ b/app/views/notify/pipeline_failed_email.text.erb @@ -35,7 +35,7 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>. Stage: <%= build.stage %> Name: <%= build.name %> <% if build.has_trace? -%> -Trace: <%= build.trace_with_state(last_lines: 10)[:text] %> +Trace: <%= build.trace.raw(last_lines: 10) %> <% end -%> <% end -%> diff --git a/app/views/projects/builds/_sidebar.html.haml b/app/views/projects/builds/_sidebar.html.haml index 6f45d5b06891399a938b18935f0bfeb71be10866..f4a66398c85faabc3d094dc8c25c3d0524a7b380 100644 --- a/app/views/projects/builds/_sidebar.html.haml +++ b/app/views/projects/builds/_sidebar.html.haml @@ -68,7 +68,7 @@ - elsif @build.runner \##{@build.runner.id} .btn-group.btn-group-justified{ role: :group } - - if @build.has_trace_file? + - if @build.has_trace? = link_to 'Raw', raw_namespace_project_build_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' - if @build.active? = link_to "Cancel", cancel_namespace_project_build_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default', method: :post diff --git a/app/views/projects/builds/show.html.haml b/app/views/projects/builds/show.html.haml index affde75439130ef3f0823fe8520edae1d5b25cfd..0765cd2444f6572f0cf30dc7531b8fc38acd173c 100644 --- a/app/views/projects/builds/show.html.haml +++ b/app/views/projects/builds/show.html.haml @@ -73,6 +73,11 @@ = custom_icon('scroll_down_hover_active') #up-build-trace %pre.build-trace#build-trace + .js-truncated-info.truncated-info.hidden + %span< + Showing last + %span.js-truncated-info-size>< + Kb of log %code.bash.js-build-output .build-loader-animation.js-build-refresh diff --git a/db/fixtures/development/14_pipelines.rb b/db/fixtures/development/14_pipelines.rb index 534847a71079c212f8af3951af066fe40d1b26ed..3c42f7db6d587ff6e88506feda7534710c3f6ecf 100644 --- a/db/fixtures/development/14_pipelines.rb +++ b/db/fixtures/development/14_pipelines.rb @@ -130,7 +130,7 @@ def setup_artifacts(build) def setup_build_log(build) if %w(running success failed).include?(build.status) - build.trace = FFaker::Lorem.paragraphs(6).join("\n\n") + build.trace.set(FFaker::Lorem.paragraphs(6).join("\n\n")) end end diff --git a/features/steps/project/builds/summary.rb b/features/steps/project/builds/summary.rb index 19ff92f6dc6e4dbfb7939044e6c5ab486fa8fbf2..124582de6b9b715ccdefa0d7bf922584b21ff3b0 100644 --- a/features/steps/project/builds/summary.rb +++ b/features/steps/project/builds/summary.rb @@ -22,9 +22,9 @@ class Spinach::Features::ProjectBuildsSummary < Spinach::FeatureSteps end step 'recent build has been erased' do + expect(@build).not_to have_trace expect(@build.artifacts_file.exists?).to be_falsy expect(@build.artifacts_metadata.exists?).to be_falsy - expect(@build.trace).to be_empty end step 'recent build summary does not have artifacts widget' do diff --git a/features/steps/shared/builds.rb b/features/steps/shared/builds.rb index 5bc3a1f5ac44195cd6d7a0aee5b7c9a70efb5512..5549fc255255f8d6bfd335412df7b16286907a71 100644 --- a/features/steps/shared/builds.rb +++ b/features/steps/shared/builds.rb @@ -47,7 +47,7 @@ module SharedBuilds end step 'recent build has a build trace' do - @build.trace = 'job trace' + @build.trace.set('job trace') end step 'download of build artifacts archive starts' do diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index ffab0aafe59eb90451f1cc6d6be80bbc2db964af..288b03d940ce52ec9eb3631edf83b75f29f0fb74 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -118,7 +118,7 @@ class Jobs < Grape::API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace + trace = build.trace.raw body trace end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index d288369e3627ef0ffd37e4bd0896caf065dc59e6..6fbb02cb3aa04e67088baa9f4a9077d8ebe94d36 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -115,7 +115,7 @@ class Runner < Grape::API put '/:id' do job = authenticate_job! - job.update_attributes(trace: params[:trace]) if params[:trace] + job.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: job.project.path_with_namespace) @@ -145,16 +145,14 @@ class Runner < Grape::API content_range = request.headers['Content-Range'] content_range = content_range.split('-') - current_length = job.trace_length - unless current_length == content_range[0].to_i - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + stream_size = job.trace.append(request.body.read, content_range[0].to_i) + if stream_size < 0 + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end - job.append_trace(request.body.read, content_range[0].to_i) - status 202 header 'Job-Status', job.status - header 'Range', "0-#{job.trace_length}" + header 'Range', "0-#{stream_size}" end desc 'Authorize artifacts uploading for job' do diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 6f97102c6ef172d48ac52ec2e8dfb0f63ec62278..4dd03cdf24bc904824abb99e7783299306d2bf8f 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -120,7 +120,7 @@ class Builds < Grape::API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace + trace = build.trace.raw body trace end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index b3ccad7b28d2b3241f602f16ea645b72fb2106af..1020452480a08ed71bff1bdeaaf4660f2f917fd3 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -132,34 +132,54 @@ def on_109(s) set_bg_color(9, 'l') end STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze - def convert(raw, new_state) + def convert(stream, new_state) reset_state - restore_state(raw, new_state) if new_state.present? - - start = @offset - ansi = raw[@offset..-1] + restore_state(new_state, stream) if new_state.present? + + append = false + truncated = false + + cur_offset = stream.tell + if cur_offset > @offset + @offset = cur_offset + truncated = true + else + stream.seek(@offset) + append = @offset > 0 + end + start_offset = @offset open_new_tag - s = StringScanner.new(ansi) - until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) - handle_sequence(s) - elsif s.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif s.scan(/' - else - @out << s.scan(/./m) + stream.each_line do |line| + s = StringScanner.new(line) + until s.eos? + if s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/' + else + @out << s.scan(/./m) + end + @offset += s.matched_size end - @offset += s.matched_size end close_open_tags() - { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 } + OpenStruct.new( + html: @out, + state: state, + append: append, + truncated: truncated, + offset: start_offset, + size: stream.tell - start_offset, + total: stream.size + ) end def handle_sequence(s) @@ -240,10 +260,10 @@ def state Base64.urlsafe_encode64(state.to_json) end - def restore_state(raw, new_state) + def restore_state(new_state, stream) state = Base64.urlsafe_decode64(new_state) state = JSON.parse(state, symbolize_names: true) - return if state[:offset].to_i > raw.length + return if state[:offset].to_i > stream.size STATE_PARAMS.each do |param| send("#{param}=".to_sym, state[param]) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 95cc6308c3b52522b12592cd0010576fec5afdee..67b269b330ca050431a360564d474c04a84e2ec5 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -61,7 +61,7 @@ class Builds < Grape::API update_runner_info - build.update_attributes(trace: params[:trace]) if params[:trace] + build.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: build.project.path_with_namespace) @@ -92,16 +92,14 @@ class Builds < Grape::API content_range = request.headers['Content-Range'] content_range = content_range.split('-') - current_length = build.trace_length - unless current_length == content_range[0].to_i - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + stream_size = build.trace.append(request.body.read, content_range[0].to_i) + if stream_size < 0 + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end - build.append_trace(request.body.read, content_range[0].to_i) - status 202 header 'Build-Status', build.status - header 'Range', "0-#{build.trace_length}" + header 'Range', "0-#{stream_size}" end # Authorize artifacts uploading for build - Runners only diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb new file mode 100644 index 0000000000000000000000000000000000000000..5b835bb669a69860c100a5424f268882783ffda3 --- /dev/null +++ b/lib/gitlab/ci/trace.rb @@ -0,0 +1,136 @@ +module Gitlab + module Ci + class Trace + attr_reader :job + + delegate :old_trace, to: :job + + def initialize(job) + @job = job + end + + def html(last_lines: nil) + read do |stream| + stream.html(last_lines: last_lines) + end + end + + def raw(last_lines: nil) + read do |stream| + stream.raw(last_lines: last_lines) + end + end + + def extract_coverage(regex) + read do |stream| + stream.extract_coverage(regex) + end + end + + def set(data) + write do |stream| + data = job.hide_secrets(data) + stream.set(data) + end + end + + def append(data, offset) + write do |stream| + current_length = stream.size + return -current_length unless current_length == offset + + data = job.hide_secrets(data) + stream.append(data, offset) + stream.size + end + end + + def exist? + current_path.present? || old_trace.present? + end + + def read + stream = Gitlab::Ci::Trace::Stream.new do + if current_path + File.open(current_path, "rb") + elsif old_trace + StringIO.new(old_trace) + end + end + + yield stream + ensure + stream&.close + end + + def write + stream = Gitlab::Ci::Trace::Stream.new do + File.open(ensure_path, "a+b") + end + + yield(stream).tap do + job.touch if job.needs_touch? + end + ensure + stream&.close + end + + def erase! + paths.each do |trace_path| + FileUtils.rm(trace_path, force: true) + end + + job.erase_old_trace! + end + + private + + def ensure_path + return current_path if current_path + + ensure_directory + default_path + end + + def ensure_directory + unless Dir.exist?(default_directory) + FileUtils.mkdir_p(default_directory) + end + end + + def current_path + @current_path ||= paths.find do |trace_path| + File.exist?(trace_path) + end + end + + def paths + [ + default_path, + deprecated_path + ].compact + end + + def default_directory + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project_id.to_s + ) + end + + def default_path + File.join(default_directory, "#{job.id}.log") + end + + def deprecated_path + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project.ci_id.to_s, + "#{job.id}.log" + ) if job.project&.ci_id + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb new file mode 100644 index 0000000000000000000000000000000000000000..2af94e2c60eff5b826542a1a899510275245c9dc --- /dev/null +++ b/lib/gitlab/ci/trace/stream.rb @@ -0,0 +1,119 @@ +module Gitlab + module Ci + class Trace + # This was inspired from: http://stackoverflow.com/a/10219411/1520132 + class Stream + BUFFER_SIZE = 4096 + LIMIT_SIZE = 50.kilobytes + + attr_reader :stream + + delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true + + delegate :valid?, to: :stream, as: :present?, allow_nil: true + + def initialize + @stream = yield + end + + def valid? + self.stream.present? + end + + def file? + self.path.present? + end + + def limit(last_bytes = LIMIT_SIZE) + stream_size = size + if stream_size < last_bytes + last_bytes = stream_size + end + stream.seek(-last_bytes, IO::SEEK_END) + end + + def append(data, offset) + stream.truncate(offset) + stream.seek(0, IO::SEEK_END) + stream.write(data) + stream.flush() + end + + def set(data) + truncate(0) + stream.write(data) + stream.flush() + end + + def raw(last_lines: nil) + return unless valid? + + if last_lines.to_i > 0 + read_last_lines(last_lines) + else + stream.read + end + end + + def html_with_state(state = nil) + ::Ci::Ansi2html.convert(stream, state) + end + + def html(last_lines: nil) + text = raw(last_lines: last_lines) + stream = StringIO.new(text) + ::Ci::Ansi2html.convert(stream).html + end + + def extract_coverage(regex) + return unless valid? + return unless regex + + regex = Regexp.new(regex) + + match = "" + + stream.each_line do |line| + matches = line.scan(regex) + next unless matches.is_a?(Array) + + match = matches.flatten.last + coverage = match.gsub(/\d+(\.\d+)?/).first + return coverage.to_f if coverage.present? + end + rescue + # if bad regex or something goes wrong we dont want to interrupt transition + # so we just silentrly ignore error for now + end + + private + + def read_last_lines(last_lines) + chunks = [] + pos = lines = 0 + max = stream.size + + # We want an extra line to make sure fist line has full contents + while lines <= last_lines && pos < max + pos += BUFFER_SIZE + + buf = + if pos <= max + stream.seek(-pos, IO::SEEK_END) + stream.read(BUFFER_SIZE) + else # Reached the head, read only left + stream.seek(0) + stream.read(BUFFER_SIZE - (pos - max)) + end + + lines += buf.count("\n") + chunks.unshift(buf) + end + + chunks.join.lines.last(last_lines).join + .force_encoding(Encoding.default_external) + end + end + end + end +end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb deleted file mode 100644 index 1d7ddeb3e0f02277e6ef9c2ffa20e0373e4c42dd..0000000000000000000000000000000000000000 --- a/lib/gitlab/ci/trace_reader.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Gitlab - module Ci - # This was inspired from: http://stackoverflow.com/a/10219411/1520132 - class TraceReader - BUFFER_SIZE = 4096 - - attr_accessor :path, :buffer_size - - def initialize(new_path, buffer_size: BUFFER_SIZE) - self.path = new_path - self.buffer_size = Integer(buffer_size) - end - - def read(last_lines: nil) - if last_lines - read_last_lines(last_lines) - else - File.read(path) - end - end - - def read_last_lines(max_lines) - File.open(path) do |file| - chunks = [] - pos = lines = 0 - max = file.size - - # We want an extra line to make sure fist line has full contents - while lines <= max_lines && pos < max - pos += buffer_size - - buf = if pos <= max - file.seek(-pos, IO::SEEK_END) - file.read(buffer_size) - else # Reached the head, read only left - file.seek(0) - file.read(buffer_size - (pos - max)) - end - - lines += buf.count("\n") - chunks.unshift(buf) - end - - chunks.join.lines.last(max_lines).join - .force_encoding(Encoding.default_external) - end - end - end - end -end diff --git a/lib/gitlab/ci/trace_stream.rb b/lib/gitlab/ci/trace_stream.rb new file mode 100644 index 0000000000000000000000000000000000000000..e01d5bf06d39639266e9d422539380b8680c1e3a --- /dev/null +++ b/lib/gitlab/ci/trace_stream.rb @@ -0,0 +1,111 @@ +module Gitlab + module Ci + # This was inspired from: http://stackoverflow.com/a/10219411/1520132 + class TraceStream + BUFFER_SIZE = 4096 + LIMIT_SIZE = 60 + + attr_reader :stream + + delegate :read, :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true + + def initialize + @stream = yield + end + + def use + yield self + ensure + close if valid? + end + + def valid? + self.stream&.ready? + end + + def file? + self.path.present? + end + + def limit(max_bytes = LIMIT_SIZE) + stream_size = size + if stream_size < max_bytes + max_bytes = stream_size + end + stream.seek(-max_bytes, IO::SEEK_END) + end + + def append(data, offset) + stream.truncate(offset) + stream.seek(0, IO::SEEK_END) + stream.write(data) + stream.flush() + end + + def set(data) + truncate(0) + stream.write(data) + stream.flush() + end + + def read_last_lines(max_lines) + chunks = [] + pos = lines = 0 + max = file.size + + # We want an extra line to make sure fist line has full contents + while lines <= max_lines && pos < max + pos += BUFFER_SIZE + + buf = + if pos <= max + stream.seek(-pos, IO::SEEK_END) + stream.read(BUFFER_SIZE) + else # Reached the head, read only left + stream.seek(0) + stream.read(BUFFER_SIZE - (pos - max)) + end + + lines += buf.count("\n") + chunks.unshift(buf) + end + + chunks.join.lines.last(max_lines).join + .force_encoding(Encoding.default_external) + end + + def html_with_state(state = nil) + ::Ci::Ansi2html.convert(stream, state) + end + + def html_last_lines(max_lines) + text = read_last_lines(max_lines) + stream = StringIO.new(text) + ::Ci::Ansi2html.convert(stream).html + end + + def extract_coverage(regex) + return unless valid? + return unless regex + + regex = Regexp.new(regex) + + match = "" + + stream.each_line do |line| + matches = line.scan(regex).last + match = matches.last if matches.is_a?(Array) + end + + coverage = match.gsub(/\d+(\.\d+)?/).first + + if coverage.present? + coverage.to_f + end + rescue + # if bad regex or something goes wrong we dont want to interrupt transition + # so we just silentrly ignore error for now + end + end + end +end diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb index 87a0c95c4dc13a8558438a479f41fabc35515dcf..b62def83ee43e84167d5433ea5d8fc5460b11321 100644 --- a/spec/factories/ci/builds.rb +++ b/spec/factories/ci/builds.rb @@ -111,7 +111,7 @@ trait :trace do after(:create) do |build, evaluator| - build.trace = 'BUILD TRACE' + build.trace.set('BUILD TRACE') end end diff --git a/spec/features/projects/builds_spec.rb b/spec/features/projects/builds_spec.rb index 48ffaf9ddb8597357609c5e2e13b30af733ad377..ca6460215de1c39462d39928188bb56d921d3ccc 100644 --- a/spec/features/projects/builds_spec.rb +++ b/spec/features/projects/builds_spec.rb @@ -205,21 +205,13 @@ it 'loads job trace' do expect(page).to have_content 'BUILD TRACE' - build.append_trace(' and more trace', 11) + build.trace.write do |stream| + stream.append(' and more trace', 11) + end expect(page).to have_content 'BUILD TRACE and more trace' end end - - context 'when build does not have an initial trace' do - let(:build) { create(:ci_build, pipeline: pipeline) } - - it 'loads new trace' do - build.append_trace('build trace', 0) - - expect(page).to have_content 'build trace' - end - end end feature 'Variables' do @@ -401,7 +393,7 @@ it 'sends the right headers' do expect(page.status_code).to eq(200) expect(page.response_headers['Content-Type']).to eq('text/plain; charset=utf-8') - expect(page.response_headers['X-Sendfile']).to eq(build.path_to_trace) + expect(page.response_headers['X-Sendfile']).to eq(build.trace.send(:current_path)) end end @@ -420,43 +412,24 @@ context 'storage form' do let(:existing_file) { Tempfile.new('existing-trace-file').path } - let(:non_existing_file) do - file = Tempfile.new('non-existing-trace-file') - path = file.path - file.unlink - path - end - context 'when build has trace in file' do - before do - Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile') - build.run! - visit namespace_project_build_path(project.namespace, project, build) + before do + Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile') - allow_any_instance_of(Project).to receive(:ci_id).and_return(nil) - allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(existing_file) - allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(non_existing_file) + build.run! - page.within('.js-build-sidebar') { click_link 'Raw' } - end + allow_any_instance_of(Gitlab::Ci::Trace).to receive(:paths) + .and_return(paths) - it 'sends the right headers' do - expect(page.status_code).to eq(200) - expect(page.response_headers['Content-Type']).to eq('text/plain; charset=utf-8') - expect(page.response_headers['X-Sendfile']).to eq(existing_file) - end + visit namespace_project_build_path(project.namespace, project, build) end - context 'when build has trace in old file' do - before do - Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile') - build.run! - visit namespace_project_build_path(project.namespace, project, build) - - allow_any_instance_of(Project).to receive(:ci_id).and_return(999) - allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(non_existing_file) - allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(existing_file) + context 'when build has trace in file' do + let(:paths) do + [existing_file] + end + before do page.within('.js-build-sidebar') { click_link 'Raw' } end @@ -468,20 +441,10 @@ end context 'when build has trace in DB' do - before do - Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile') - build.run! - visit namespace_project_build_path(project.namespace, project, build) - - allow_any_instance_of(Project).to receive(:ci_id).and_return(nil) - allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(non_existing_file) - allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(non_existing_file) - - page.within('.js-build-sidebar') { click_link 'Raw' } - end + let(:paths) { [] } it 'sends the right headers' do - expect(page.status_code).to eq(404) + expect(page.status_code).not_to have_link('Raw') end end end diff --git a/spec/javascripts/build_spec.js b/spec/javascripts/build_spec.js index beee6cb2969e24c154c354dc790261fae925ca5f..8b85dc721ece450d3a464f3bbe8c6eb958a1ef44 100644 --- a/spec/javascripts/build_spec.js +++ b/spec/javascripts/build_spec.js @@ -93,27 +93,31 @@ describe('Build', () => { describe('running build', () => { beforeEach(function () { - $('.js-build-options').data('buildStatus', 'running'); this.build = new Build(); - spyOn(this.build, 'location').and.returnValue(BUILD_URL); }); it('updates the build trace on an interval', function () { + spyOn(gl.utils, 'visitUrl'); + jasmine.clock().tick(4001); - expect($.ajax.calls.count()).toBe(2); - let [{ url, dataType, success, context }] = $.ajax.calls.argsFor(1); - expect(url).toBe( - `${BUILD_URL}/trace.json?state=`, - ); - expect(dataType).toBe('json'); - expect(success).toEqual(jasmine.any(Function)); + expect($.ajax.calls.count()).toBe(1); - success.call(context, { + // We have to do it this way to prevent Webpack to fail to compile + // when destructuring assignments and reusing + // the same variables names inside the same scope + let args = $.ajax.calls.argsFor(0)[0]; + + expect(args.url).toBe(`${BUILD_URL}/trace.json`); + expect(args.dataType).toBe('json'); + expect(args.success).toEqual(jasmine.any(Function)); + + args.success.call($, { html: 'Update', status: 'running', state: 'newstate', append: true, + complete: false, }); expect($('#build-trace .js-build-output').text()).toMatch(/Update/); @@ -121,17 +125,20 @@ describe('Build', () => { jasmine.clock().tick(4001); - expect($.ajax.calls.count()).toBe(3); - [{ url, dataType, success, context }] = $.ajax.calls.argsFor(2); - expect(url).toBe(`${BUILD_URL}/trace.json?state=newstate`); - expect(dataType).toBe('json'); - expect(success).toEqual(jasmine.any(Function)); + expect($.ajax.calls.count()).toBe(2); + + args = $.ajax.calls.argsFor(1)[0]; + expect(args.url).toBe(`${BUILD_URL}/trace.json`); + expect(args.dataType).toBe('json'); + expect(args.data.state).toBe('newstate'); + expect(args.success).toEqual(jasmine.any(Function)); - success.call(context, { + args.success.call($, { html: 'More', status: 'running', state: 'finalstate', append: true, + complete: true, }); expect($('#build-trace .js-build-output').text()).toMatch(/UpdateMore/); @@ -139,19 +146,22 @@ describe('Build', () => { }); it('replaces the entire build trace', () => { + spyOn(gl.utils, 'visitUrl'); + jasmine.clock().tick(4001); - let [{ success, context }] = $.ajax.calls.argsFor(1); - success.call(context, { + let args = $.ajax.calls.argsFor(0)[0]; + args.success.call($, { html: 'Update', status: 'running', - append: true, + append: false, + complete: false, }); expect($('#build-trace .js-build-output').text()).toMatch(/Update/); jasmine.clock().tick(4001); - [{ success, context }] = $.ajax.calls.argsFor(2); - success.call(context, { + args = $.ajax.calls.argsFor(1)[0]; + args.success.call($, { html: 'Different', status: 'running', append: false, @@ -161,15 +171,34 @@ describe('Build', () => { expect($('#build-trace .js-build-output').text()).toMatch(/Different/); }); + it('shows information about truncated log', () => { + jasmine.clock().tick(4001); + const [{ success }] = $.ajax.calls.argsFor(0); + + success.call($, { + html: 'Update', + status: 'success', + append: false, + truncated: true, + size: '50', + }); + + expect( + $('#build-trace .js-truncated-info').text().trim(), + ).toContain('Showing last 50 Kb of log'); + expect($('#build-trace .js-truncated-info-size').text()).toMatch('50'); + }); + it('reloads the page when the build is done', () => { spyOn(gl.utils, 'visitUrl'); jasmine.clock().tick(4001); - const [{ success, context }] = $.ajax.calls.argsFor(1); - success.call(context, { + const [{ success }] = $.ajax.calls.argsFor(0); + success.call($, { html: 'Final', status: 'passed', append: true, + complete: true, }); expect(gl.utils.visitUrl).toHaveBeenCalledWith(BUILD_URL); diff --git a/spec/lib/ci/ansi2html_spec.rb b/spec/lib/ci/ansi2html_spec.rb index 0762fd7e56a007034af3c0f7e9bba10e66573d5f..a5dfb49478a458245866f5477bd4b81d16eb79da 100644 --- a/spec/lib/ci/ansi2html_spec.rb +++ b/spec/lib/ci/ansi2html_spec.rb @@ -1,159 +1,160 @@ require 'spec_helper' describe Ci::Ansi2html, lib: true do - subject { Ci::Ansi2html } + subject { described_class } it "prints non-ansi as-is" do - expect(subject.convert("Hello")[:html]).to eq('Hello') + expect(convert_html("Hello")).to eq('Hello') end it "strips non-color-changing controll sequences" do - expect(subject.convert("Hello \e[2Kworld")[:html]).to eq('Hello world') + expect(convert_html("Hello \e[2Kworld")).to eq('Hello world') end it "prints simply red" do - expect(subject.convert("\e[31mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[31mHello\e[0m")).to eq('Hello') end it "prints simply red without trailing reset" do - expect(subject.convert("\e[31mHello")[:html]).to eq('Hello') + expect(convert_html("\e[31mHello")).to eq('Hello') end it "prints simply yellow" do - expect(subject.convert("\e[33mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[33mHello\e[0m")).to eq('Hello') end it "prints default on blue" do - expect(subject.convert("\e[39;44mHello")[:html]).to eq('Hello') + expect(convert_html("\e[39;44mHello")).to eq('Hello') end it "prints red on blue" do - expect(subject.convert("\e[31;44mHello")[:html]).to eq('Hello') + expect(convert_html("\e[31;44mHello")).to eq('Hello') end it "resets colors after red on blue" do - expect(subject.convert("\e[31;44mHello\e[0m world")[:html]).to eq('Hello world') + expect(convert_html("\e[31;44mHello\e[0m world")).to eq('Hello world') end it "performs color change from red/blue to yellow/blue" do - expect(subject.convert("\e[31;44mHello \e[33mworld")[:html]).to eq('Hello world') + expect(convert_html("\e[31;44mHello \e[33mworld")).to eq('Hello world') end it "performs color change from red/blue to yellow/green" do - expect(subject.convert("\e[31;44mHello \e[33;42mworld")[:html]).to eq('Hello world') + expect(convert_html("\e[31;44mHello \e[33;42mworld")).to eq('Hello world') end it "performs color change from red/blue to reset to yellow/green" do - expect(subject.convert("\e[31;44mHello\e[0m \e[33;42mworld")[:html]).to eq('Hello world') + expect(convert_html("\e[31;44mHello\e[0m \e[33;42mworld")).to eq('Hello world') end it "ignores unsupported codes" do - expect(subject.convert("\e[51mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[51mHello\e[0m")).to eq('Hello') end it "prints light red" do - expect(subject.convert("\e[91mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[91mHello\e[0m")).to eq('Hello') end it "prints default on light red" do - expect(subject.convert("\e[101mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[101mHello\e[0m")).to eq('Hello') end it "performs color change from red/blue to default/blue" do - expect(subject.convert("\e[31;44mHello \e[39mworld")[:html]).to eq('Hello world') + expect(convert_html("\e[31;44mHello \e[39mworld")).to eq('Hello world') end it "performs color change from light red/blue to default/blue" do - expect(subject.convert("\e[91;44mHello \e[39mworld")[:html]).to eq('Hello world') + expect(convert_html("\e[91;44mHello \e[39mworld")).to eq('Hello world') end it "prints bold text" do - expect(subject.convert("\e[1mHello")[:html]).to eq('Hello') + expect(convert_html("\e[1mHello")).to eq('Hello') end it "resets bold text" do - expect(subject.convert("\e[1mHello\e[21m world")[:html]).to eq('Hello world') - expect(subject.convert("\e[1mHello\e[22m world")[:html]).to eq('Hello world') + expect(convert_html("\e[1mHello\e[21m world")).to eq('Hello world') + expect(convert_html("\e[1mHello\e[22m world")).to eq('Hello world') end it "prints italic text" do - expect(subject.convert("\e[3mHello")[:html]).to eq('Hello') + expect(convert_html("\e[3mHello")).to eq('Hello') end it "resets italic text" do - expect(subject.convert("\e[3mHello\e[23m world")[:html]).to eq('Hello world') + expect(convert_html("\e[3mHello\e[23m world")).to eq('Hello world') end it "prints underlined text" do - expect(subject.convert("\e[4mHello")[:html]).to eq('Hello') + expect(convert_html("\e[4mHello")).to eq('Hello') end it "resets underlined text" do - expect(subject.convert("\e[4mHello\e[24m world")[:html]).to eq('Hello world') + expect(convert_html("\e[4mHello\e[24m world")).to eq('Hello world') end it "prints concealed text" do - expect(subject.convert("\e[8mHello")[:html]).to eq('Hello') + expect(convert_html("\e[8mHello")).to eq('Hello') end it "resets concealed text" do - expect(subject.convert("\e[8mHello\e[28m world")[:html]).to eq('Hello world') + expect(convert_html("\e[8mHello\e[28m world")).to eq('Hello world') end it "prints crossed-out text" do - expect(subject.convert("\e[9mHello")[:html]).to eq('Hello') + expect(convert_html("\e[9mHello")).to eq('Hello') end it "resets crossed-out text" do - expect(subject.convert("\e[9mHello\e[29m world")[:html]).to eq('Hello world') + expect(convert_html("\e[9mHello\e[29m world")).to eq('Hello world') end it "can print 256 xterm fg colors" do - expect(subject.convert("\e[38;5;16mHello")[:html]).to eq('Hello') + expect(convert_html("\e[38;5;16mHello")).to eq('Hello') end it "can print 256 xterm fg colors on normal magenta background" do - expect(subject.convert("\e[38;5;16;45mHello")[:html]).to eq('Hello') + expect(convert_html("\e[38;5;16;45mHello")).to eq('Hello') end it "can print 256 xterm bg colors" do - expect(subject.convert("\e[48;5;240mHello")[:html]).to eq('Hello') + expect(convert_html("\e[48;5;240mHello")).to eq('Hello') end it "can print 256 xterm bg colors on normal magenta foreground" do - expect(subject.convert("\e[48;5;16;35mHello")[:html]).to eq('Hello') + expect(convert_html("\e[48;5;16;35mHello")).to eq('Hello') end it "prints bold colored text vividly" do - expect(subject.convert("\e[1;31mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[1;31mHello\e[0m")).to eq('Hello') end it "prints bold light colored text correctly" do - expect(subject.convert("\e[1;91mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[1;91mHello\e[0m")).to eq('Hello') end it "prints <" do - expect(subject.convert("<")[:html]).to eq('<') + expect(convert_html("<")).to eq('<') end it "replaces newlines with line break tags" do - expect(subject.convert("\n")[:html]).to eq('
') + expect(convert_html("\n")).to eq('
') end it "groups carriage returns with newlines" do - expect(subject.convert("\r\n")[:html]).to eq('
') + expect(convert_html("\r\n")).to eq('
') end describe "incremental update" do shared_examples 'stateable converter' do - let(:pass1) { subject.convert(pre_text) } - let(:pass2) { subject.convert(pre_text + text, pass1[:state]) } + let(:pass1_stream) { StringIO.new(pre_text) } + let(:pass2_stream) { StringIO.new(pre_text + text) } + let(:pass1) { subject.convert(pass1_stream) } + let(:pass2) { subject.convert(pass2_stream, pass1.state) } it "to returns html to append" do - expect(pass2[:append]).to be_truthy - expect(pass2[:html]).to eq(html) - expect(pass1[:text] + pass2[:text]).to eq(pre_text + text) - expect(pass1[:html] + pass2[:html]).to eq(pre_html + html) + expect(pass2.append).to be_truthy + expect(pass2.html).to eq(html) + expect(pass1.html + pass2.html).to eq(pre_html + html) end end @@ -193,4 +194,27 @@ it_behaves_like 'stateable converter' end end + + describe "truncates" do + let(:text) { "Hello World" } + let(:stream) { StringIO.new(text) } + let(:subject) { described_class.convert(stream) } + + before do + stream.seek(3, IO::SEEK_SET) + end + + it "returns truncated output" do + expect(subject.truncated).to be_truthy + end + + it "does not append output" do + expect(subject.append).to be_falsey + end + end + + def convert_html(data) + stream = StringIO.new(data) + subject.convert(stream).html + end end diff --git a/spec/lib/gitlab/ci/trace/stream_spec.rb b/spec/lib/gitlab/ci/trace/stream_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..f1a1a71c52815b92470384ec86443f12133d0dee --- /dev/null +++ b/spec/lib/gitlab/ci/trace/stream_spec.rb @@ -0,0 +1,201 @@ +require 'spec_helper' + +describe Gitlab::Ci::Trace::Stream do + describe 'delegates' do + subject { described_class.new { nil } } + + it { is_expected.to delegate_method(:close).to(:stream) } + it { is_expected.to delegate_method(:tell).to(:stream) } + it { is_expected.to delegate_method(:seek).to(:stream) } + it { is_expected.to delegate_method(:size).to(:stream) } + it { is_expected.to delegate_method(:path).to(:stream) } + it { is_expected.to delegate_method(:truncate).to(:stream) } + it { is_expected.to delegate_method(:valid?).to(:stream).as(:present?) } + it { is_expected.to delegate_method(:file?).to(:path).as(:present?) } + end + + describe '#limit' do + let(:stream) do + described_class.new do + StringIO.new("12345678") + end + end + + it 'if size is larger we start from beggining' do + stream.limit(10) + + expect(stream.tell).to eq(0) + end + + it 'if size is smaller we start from the end' do + stream.limit(2) + + expect(stream.tell).to eq(6) + end + end + + describe '#append' do + let(:stream) do + described_class.new do + StringIO.new("12345678") + end + end + + it "truncates and append content" do + stream.append("89", 4) + stream.seek(0) + + expect(stream.size).to eq(6) + expect(stream.raw).to eq("123489") + end + end + + describe '#set' do + let(:stream) do + described_class.new do + StringIO.new("12345678") + end + end + + before do + stream.set("8901") + end + + it "overwrite content" do + stream.seek(0) + + expect(stream.size).to eq(4) + expect(stream.raw).to eq("8901") + end + end + + describe '#raw' do + let(:path) { __FILE__ } + let(:lines) { File.readlines(path) } + let(:stream) do + described_class.new do + File.open(path) + end + end + + it 'returns all contents if last_lines is not specified' do + result = stream.raw + + expect(result).to eq(lines.join) + expect(result.encoding).to eq(Encoding.default_external) + end + + context 'limit max lines' do + before do + # specifying BUFFER_SIZE forces to seek backwards + allow(described_class).to receive(:BUFFER_SIZE) + .and_return(2) + end + + it 'returns last few lines' do + result = stream.raw(last_lines: 2) + + expect(result).to eq(lines.last(2).join) + expect(result.encoding).to eq(Encoding.default_external) + end + + it 'returns everything if trying to get too many lines' do + result = stream.raw(last_lines: lines.size * 2) + + expect(result).to eq(lines.join) + expect(result.encoding).to eq(Encoding.default_external) + end + end + end + + describe '#html_with_state' do + let(:stream) do + described_class.new do + StringIO.new("1234") + end + end + + it 'returns html content with state' do + result = stream.html_with_state + + expect(result.html).to eq("1234") + end + + context 'follow-up state' do + let!(:last_result) { stream.html_with_state } + + before do + stream.append("5678", 4) + stream.seek(0) + end + + it "returns appended trace" do + result = stream.html_with_state(last_result.state) + + expect(result.append).to be_truthy + expect(result.html).to eq("5678") + end + end + end + + describe '#html' do + let(:stream) do + described_class.new do + StringIO.new("12\n34\n56") + end + end + + it "returns html" do + expect(stream.html).to eq("12
34
56") + end + + it "returns html for last line only" do + expect(stream.html(last_lines: 1)).to eq("56") + end + end + + describe '#extract_coverage' do + let(:stream) do + described_class.new do + StringIO.new(data) + end + end + + subject { stream.extract_coverage(regex) } + + context 'valid content & regex' do + let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered' } + let(:regex) { '\(\d+.\d+\%\) covered' } + + it { is_expected.to eq(98.29) } + end + + context 'valid content & bad regex' do + let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' } + let(:regex) { 'very covered' } + + it { is_expected.to be_nil } + end + + context 'no coverage content & regex' do + let(:data) { 'No coverage for today :sad:' } + let(:regex) { '\(\d+.\d+\%\) covered' } + + it { is_expected.to be_nil } + end + + context 'multiple results in content & regex' do + let(:data) { ' (98.39%) covered. (98.29%) covered' } + let(:regex) { '\(\d+.\d+\%\) covered' } + + it { is_expected.to eq(98.29) } + end + + context 'using a regex capture' do + let(:data) { 'TOTAL 9926 3489 65%' } + let(:regex) { 'TOTAL\s+\d+\s+\d+\s+(\d{1,3}\%)' } + + it { is_expected.to eq(65) } + end + end +end diff --git a/spec/lib/gitlab/ci/trace_reader_spec.rb b/spec/lib/gitlab/ci/trace_reader_spec.rb index ff5551bf703e1f77baa1378366ba19691b492aeb..bec08bc2af0666a0f28dc6341f97a83792d64458 100644 --- a/spec/lib/gitlab/ci/trace_reader_spec.rb +++ b/spec/lib/gitlab/ci/trace_reader_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Ci::TraceReader do +describe Gitlab::Ci::Trace do let(:path) { __FILE__ } let(:lines) { File.readlines(path) } let(:bytesize) { lines.sum(&:bytesize) } diff --git a/spec/lib/gitlab/ci/trace_spec.rb b/spec/lib/gitlab/ci/trace_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..69e8dc9220d51abc69d0c9b12c22cfbcf81f3c2f --- /dev/null +++ b/spec/lib/gitlab/ci/trace_spec.rb @@ -0,0 +1,216 @@ +require 'spec_helper' + +describe Gitlab::Ci::Trace do + let(:build) { create(:ci_build) } + let(:trace) { described_class.new(build) } + + describe "associations" do + it { expect(trace).to respond_to(:job) } + it { expect(trace).to delegate_method(:old_trace).to(:job) } + end + + describe '#html' do + before do + trace.set("12\n34") + end + + it "returns formatted html" do + expect(trace.html).to eq("12
34") + end + + it "returns last line of formatted html" do + expect(trace.html(last_lines: 1)).to eq("34") + end + end + + describe '#raw' do + before do + trace.set("12\n34") + end + + it "returns raw output" do + expect(trace.raw).to eq("12\n34") + end + + it "returns last line of raw output" do + expect(trace.raw(last_lines: 1)).to eq("34") + end + end + + describe '#extract_coverage' do + let(:regex) { '\(\d+.\d+\%\) covered' } + + before do + trace.set('Coverage 1033 / 1051 LOC (98.29%) covered') + end + + it "returns valid coverage" do + expect(trace.extract_coverage(regex)).to eq(98.29) + end + end + + describe '#set' do + before do + trace.set("12") + end + + it "returns trace" do + expect(trace.raw).to eq("12") + end + + context 'overwrite trace' do + before do + trace.set("34") + end + + it "returns new trace" do + expect(trace.raw).to eq("34") + end + end + + context 'runners token' do + let(:token) { 'my_secret_token' } + + before do + build.project.update(runners_token: token) + trace.set(token) + end + + it "hides token" do + expect(trace.raw).not_to include(token) + end + end + + context 'hides build token' do + let(:token) { 'my_secret_token' } + + before do + build.update(token: token) + trace.set(token) + end + + it "hides token" do + expect(trace.raw).not_to include(token) + end + end + end + + describe '#append' do + before do + trace.set("1234") + end + + it "returns correct trace" do + expect(trace.append("56", 4)).to eq(6) + expect(trace.raw).to eq("123456") + end + + context 'tries to append trace at different offset' do + it "fails with append" do + expect(trace.append("56", 2)).to eq(-4) + expect(trace.raw).to eq("1234") + end + end + + context 'runners token' do + let(:token) { 'my_secret_token' } + + before do + build.project.update(runners_token: token) + trace.append(token, 0) + end + + it "hides token" do + expect(trace.raw).not_to include(token) + end + end + + context 'build token' do + let(:token) { 'my_secret_token' } + + before do + build.update(token: token) + trace.append(token, 0) + end + + it "hides token" do + expect(trace.raw).not_to include(token) + end + end + end + + describe 'trace handling' do + context 'trace does not exist' do + it { expect(trace.exist?).to be(false) } + end + + context 'new trace path is used' do + before do + trace.send(:ensure_directory) + + File.open(trace.send(:default_path), "w") do |file| + file.write("data") + end + end + + it "trace exist" do + expect(trace.exist?).to be(true) + end + + it "can be erased" do + trace.erase! + expect(trace.exist?).to be(false) + end + end + + context 'deprecated path' do + let(:path) { trace.send(:deprecated_path) } + + context 'with valid ci_id' do + before do + build.project.update(ci_id: 1000) + + FileUtils.mkdir_p(File.dirname(path)) + + File.open(path, "w") do |file| + file.write("data") + end + end + + it "trace exist" do + expect(trace.exist?).to be(true) + end + + it "can be erased" do + trace.erase! + expect(trace.exist?).to be(false) + end + end + + context 'without valid ci_id' do + it "does not return deprecated path" do + expect(path).to be_nil + end + end + end + + context 'stored in database' do + before do + build.send(:write_attribute, :trace, "data") + end + + it "trace exist" do + expect(trace.exist?).to be(true) + end + + it "can be erased" do + trace.erase! + expect(trace.exist?).to be(false) + end + + it "returns database data" do + expect(trace.raw).to eq("data") + end + end + end +end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index ac47b34b6fc8cb48fddc8383eb1fc74ce0406e02..8601160561f7615206d3838f79bc964bf9cc02c6 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -17,8 +17,9 @@ it { is_expected.to belong_to(:trigger_request) } it { is_expected.to belong_to(:erased_by) } it { is_expected.to have_many(:deployments) } - it { is_expected.to validate_presence_of :ref } - it { is_expected.to respond_to :trace_html } + it { is_expected.to validate_presence_of(:ref) } + it { is_expected.to respond_to(:has_trace?) } + it { is_expected.to respond_to(:trace) } describe '#actionize' do context 'when build is a created' do @@ -78,32 +79,6 @@ end end - describe '#append_trace' do - subject { build.trace_html } - - context 'when build.trace hides runners token' do - let(:token) { 'my_secret_token' } - - before do - build.project.update(runners_token: token) - build.append_trace(token, 0) - end - - it { is_expected.not_to include(token) } - end - - context 'when build.trace hides build token' do - let(:token) { 'my_secret_token' } - - before do - build.update(token: token) - build.append_trace(token, 0) - end - - it { is_expected.not_to include(token) } - end - end - describe '#artifacts?' do subject { build.artifacts? } @@ -272,12 +247,98 @@ describe '#update_coverage' do context "regarding coverage_regex's value," do - it "saves the correct extracted coverage value" do + before do build.coverage_regex = '\(\d+.\d+\%\) covered' - allow(build).to receive(:trace) { 'Coverage 1033 / 1051 LOC (98.29%) covered' } - expect(build).to receive(:update_attributes).with(coverage: 98.29) { true } - expect(build.update_coverage).to be true + build.trace.set('Coverage 1033 / 1051 LOC (98.29%) covered') + end + + it "saves the correct extracted coverage value" do + expect(build.update_coverage).to be(true) + expect(build.coverage).to eq(98.29) + end + end + end + + describe '#trace' do + subject { build.trace } + + it { is_expected.to be_a(Gitlab::Ci::Trace) } + end + + describe '#has_trace?' do + subject { build.has_trace? } + + it "expect to call exist? method" do + expect_any_instance_of(Gitlab::Ci::Trace).to receive(:exist?) + .and_return(true) + + is_expected.to be(true) + end + end + + describe '#trace=' do + it "expect to fail trace=" do + expect { build.trace = "new" }.to raise_error(NotImplementedError) + end + end + + describe '#old_trace' do + subject { build.old_trace } + + before do + build.update_column(:trace, 'old trace') + end + + it "expect to receive data from database" do + is_expected.to eq('old trace') + end + end + + describe '#erase_old_trace!' do + subject { build.send(:read_attribute, :trace) } + + before do + build.send(:write_attribute, :trace, 'old trace') + end + + it "expect to receive data from database" do + build.erase_old_trace! + + is_expected.to be_nil + end + end + + describe '#hide_secrets' do + let(:subject) { build.hide_secrets(data) } + + context 'hide runners token' do + let(:data) { 'new token data'} + + before do + build.project.update(runners_token: 'token') end + + it { is_expected.to eq('new xxxxx data') } + end + + context 'hide build token' do + let(:data) { 'new token data'} + + before do + build.update(token: 'token') + end + + it { is_expected.to eq('new xxxxx data') } + end + + context 'hide build token' do + let(:data) { 'new token data'} + + before do + build.update(token: 'token') + end + + it { is_expected.to eq('new xxxxx data') } end end @@ -438,7 +499,7 @@ end it 'erases build trace in trace file' do - expect(build.trace).to be_empty + expect(build).not_to have_trace end it 'sets erased to true' do @@ -532,38 +593,6 @@ end end - describe '#extract_coverage' do - context 'valid content & regex' do - subject { build.extract_coverage('Coverage 1033 / 1051 LOC (98.29%) covered', '\(\d+.\d+\%\) covered') } - - it { is_expected.to eq(98.29) } - end - - context 'valid content & bad regex' do - subject { build.extract_coverage('Coverage 1033 / 1051 LOC (98.29%) covered', 'very covered') } - - it { is_expected.to be_nil } - end - - context 'no coverage content & regex' do - subject { build.extract_coverage('No coverage for today :sad:', '\(\d+.\d+\%\) covered') } - - it { is_expected.to be_nil } - end - - context 'multiple results in content & regex' do - subject { build.extract_coverage(' (98.39%) covered. (98.29%) covered', '\(\d+.\d+\%\) covered') } - - it { is_expected.to eq(98.29) } - end - - context 'using a regex capture' do - subject { build.extract_coverage('TOTAL 9926 3489 65%', 'TOTAL\s+\d+\s+\d+\s+(\d{1,3}\%)') } - - it { is_expected.to eq(65) } - end - end - describe '#first_pending' do let!(:first) { create(:ci_build, pipeline: pipeline, status: 'pending', created_at: Date.yesterday) } let!(:second) { create(:ci_build, pipeline: pipeline, status: 'pending') } @@ -983,32 +1012,6 @@ def create_mr(build, pipeline, factory: :merge_request, created_at: Time.now) it { is_expected.to eq(project.name) } end - describe '#raw_trace' do - subject { build.raw_trace } - - context 'when build.trace hides runners token' do - let(:token) { 'my_secret_token' } - - before do - build.project.update(runners_token: token) - build.update(trace: token) - end - - it { is_expected.not_to include(token) } - end - - context 'when build.trace hides build token' do - let(:token) { 'my_secret_token' } - - before do - build.update(token: token) - build.update(trace: token) - end - - it { is_expected.not_to include(token) } - end - end - describe '#ref_slug' do { 'master' => 'master', @@ -1074,61 +1077,6 @@ def create_mr(build, pipeline, factory: :merge_request, created_at: Time.now) end end - describe '#trace' do - it 'obfuscates project runners token' do - allow(build).to receive(:raw_trace).and_return("Test: #{build.project.runners_token}") - - expect(build.trace).to eq("Test: xxxxxxxxxxxxxxxxxxxx") - end - - it 'empty project runners token' do - allow(build).to receive(:raw_trace).and_return(test_trace) - # runners_token can't normally be set to nil - allow(build.project).to receive(:runners_token).and_return(nil) - - expect(build.trace).to eq(test_trace) - end - - context 'when build does not have trace' do - it 'is is empty' do - expect(build.trace).to be_nil - end - end - - context 'when trace contains text' do - let(:text) { 'example output' } - before do - build.trace = text - end - - it { expect(build.trace).to eq(text) } - end - - context 'when trace hides runners token' do - let(:token) { 'my_secret_token' } - - before do - build.update(trace: token) - build.project.update(runners_token: token) - end - - it { expect(build.trace).not_to include(token) } - it { expect(build.raw_trace).to include(token) } - end - - context 'when build.trace hides build token' do - let(:token) { 'my_secret_token' } - - before do - build.update(trace: token) - build.update(token: token) - end - - it { expect(build.trace).not_to include(token) } - it { expect(build.raw_trace).to include(token) } - end - end - describe '#has_expiring_artifacts?' do context 'when artifacts have expiration date set' do before { build.update(artifacts_expire_at: 1.day.from_now) } @@ -1147,66 +1095,6 @@ def create_mr(build, pipeline, factory: :merge_request, created_at: Time.now) end end - describe '#has_trace_file?' do - context 'when there is no trace' do - it { expect(build.has_trace_file?).to be_falsey } - it { expect(build.trace).to be_nil } - end - - context 'when there is a trace' do - context 'when trace is stored in file' do - let(:build_with_trace) { create(:ci_build, :trace) } - - it { expect(build_with_trace.has_trace_file?).to be_truthy } - it { expect(build_with_trace.trace).to eq('BUILD TRACE') } - end - - context 'when trace is stored in old file' do - before do - allow(build.project).to receive(:ci_id).and_return(999) - allow(File).to receive(:exist?).with(build.path_to_trace).and_return(false) - allow(File).to receive(:exist?).with(build.old_path_to_trace).and_return(true) - allow(File).to receive(:read).with(build.old_path_to_trace).and_return(test_trace) - end - - it { expect(build.has_trace_file?).to be_truthy } - it { expect(build.trace).to eq(test_trace) } - end - - context 'when trace is stored in DB' do - before do - allow(build.project).to receive(:ci_id).and_return(nil) - allow(build).to receive(:read_attribute).with(:trace).and_return(test_trace) - allow(File).to receive(:exist?).with(build.path_to_trace).and_return(false) - allow(File).to receive(:exist?).with(build.old_path_to_trace).and_return(false) - end - - it { expect(build.has_trace_file?).to be_falsey } - it { expect(build.trace).to eq(test_trace) } - end - end - end - - describe '#trace_file_path' do - context 'when trace is stored in file' do - before do - allow(build).to receive(:has_trace_file?).and_return(true) - allow(build).to receive(:has_old_trace_file?).and_return(false) - end - - it { expect(build.trace_file_path).to eq(build.path_to_trace) } - end - - context 'when trace is stored in old file' do - before do - allow(build).to receive(:has_trace_file?).and_return(true) - allow(build).to receive(:has_old_trace_file?).and_return(true) - end - - it { expect(build.trace_file_path).to eq(build.old_path_to_trace) } - end - end - describe '#update_project_statistics' do let!(:build) { create(:ci_build, artifacts_size: 23) } diff --git a/spec/requests/api/jobs_spec.rb b/spec/requests/api/jobs_spec.rb index 9450701064b5f4c1a668fbe3322b29d96bb223f2..d8a56c02a638ab75de2142f17b8a739b917bf05d 100644 --- a/spec/requests/api/jobs_spec.rb +++ b/spec/requests/api/jobs_spec.rb @@ -320,7 +320,7 @@ def get_for_ref(ref = pipeline.ref, job = build.name) context 'authorized user' do it 'returns specific job trace' do expect(response).to have_http_status(200) - expect(response.body).to eq(build.trace) + expect(response.body).to eq(build.trace.raw) end end @@ -408,7 +408,7 @@ def get_for_ref(ref = pipeline.ref, job = build.name) it 'erases job content' do expect(response).to have_http_status(201) - expect(build.trace).to be_empty + expect(build).not_to have_trace expect(build.artifacts_file.exists?).to be_falsy expect(build.artifacts_metadata.exists?).to be_falsy end diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index 1cfac7353d4ed536e6c250a48ac68178982eb3b2..409a59d6c23261d9bc7001b6cb635c285ff4431b 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -592,7 +592,7 @@ def request_job(token = runner.token, **params) update_job(trace: 'BUILD TRACE UPDATED') expect(response).to have_http_status(200) - expect(job.reload.trace).to eq 'BUILD TRACE UPDATED' + expect(job.reload.trace.raw).to eq 'BUILD TRACE UPDATED' end end @@ -600,7 +600,7 @@ def request_job(token = runner.token, **params) it 'does not override trace information' do update_job - expect(job.reload.trace).to eq 'BUILD TRACE' + expect(job.reload.trace.raw).to eq 'BUILD TRACE' end end @@ -631,7 +631,7 @@ def update_job(token = job.token, **params) context 'when request is valid' do it 'gets correct response' do expect(response.status).to eq 202 - expect(job.reload.trace).to eq 'BUILD TRACE appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended' expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Job-Status' end @@ -642,7 +642,7 @@ def update_job(token = job.token, **params) it "changes the job's trace" do patch_the_trace - expect(job.reload.trace).to eq 'BUILD TRACE appended appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended' end context 'when Runner makes a force-patch' do @@ -651,7 +651,7 @@ def update_job(token = job.token, **params) it "doesn't change the build.trace" do force_patch_the_trace - expect(job.reload.trace).to eq 'BUILD TRACE appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended' end end end @@ -664,7 +664,7 @@ def update_job(token = job.token, **params) it 'changes the job.trace' do patch_the_trace - expect(job.reload.trace).to eq 'BUILD TRACE appended appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended' end context 'when Runner makes a force-patch' do @@ -673,7 +673,7 @@ def update_job(token = job.token, **params) it "doesn't change the job.trace" do force_patch_the_trace - expect(job.reload.trace).to eq 'BUILD TRACE appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended' end end end @@ -698,7 +698,7 @@ def update_job(token = job.token, **params) it 'gets correct response' do expect(response.status).to eq 202 - expect(job.reload.trace).to eq 'BUILD TRACE appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended' expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Job-Status' end @@ -738,9 +738,11 @@ def update_job(token = job.token, **params) def patch_the_trace(content = ' appended', request_headers = nil) unless request_headers - offset = job.trace_length - limit = offset + content.length - 1 - request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) + job.trace.read do |stream| + offset = stream.size + limit = offset + content.length - 1 + request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) + end end Timecop.travel(job.updated_at + update_interval) do diff --git a/spec/requests/api/v3/builds_spec.rb b/spec/requests/api/v3/builds_spec.rb index a50c22a6dd1305ced3cb22c5551b821a204ff763..e97d2b0cee0e20ef1fe6f2b1116cb00a1b622ef3 100644 --- a/spec/requests/api/v3/builds_spec.rb +++ b/spec/requests/api/v3/builds_spec.rb @@ -330,7 +330,7 @@ def path_for_ref(ref = pipeline.ref, job = build.name) context 'authorized user' do it 'returns specific job trace' do expect(response).to have_http_status(200) - expect(response.body).to eq(build.trace) + expect(response.body).to eq(build.trace.raw) end end @@ -418,7 +418,7 @@ def path_for_ref(ref = pipeline.ref, job = build.name) it 'erases job content' do expect(response.status).to eq 201 - expect(build.trace).to be_empty + expect(build).not_to have_trace expect(build.artifacts_file.exists?).to be_falsy expect(build.artifacts_metadata.exists?).to be_falsy end diff --git a/spec/requests/ci/api/builds_spec.rb b/spec/requests/ci/api/builds_spec.rb index c879f37f50d4fac277731575044f51af68a87c78..ef30d8638dd52f533530ec3777952ca346094517 100644 --- a/spec/requests/ci/api/builds_spec.rb +++ b/spec/requests/ci/api/builds_spec.rb @@ -285,7 +285,7 @@ def register_builds(token = runner.token, **params) end it 'does not override trace information when no trace is given' do - expect(build.reload.trace).to eq 'BUILD TRACE' + expect(build.reload.trace.raw).to eq 'BUILD TRACE' end context 'job has been erased' do @@ -309,9 +309,11 @@ def register_builds(token = runner.token, **params) def patch_the_trace(content = ' appended', request_headers = nil) unless request_headers - offset = build.trace_length - limit = offset + content.length - 1 - request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) + build.trace.read do |stream| + offset = stream.size + limit = offset + content.length - 1 + request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) + end end Timecop.travel(build.updated_at + update_interval) do @@ -335,7 +337,7 @@ def force_patch_the_trace context 'when request is valid' do it 'gets correct response' do expect(response.status).to eq 202 - expect(build.reload.trace).to eq 'BUILD TRACE appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Build-Status' end @@ -346,7 +348,7 @@ def force_patch_the_trace it 'changes the build trace' do patch_the_trace - expect(build.reload.trace).to eq 'BUILD TRACE appended appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended appended' end context 'when Runner makes a force-patch' do @@ -355,7 +357,7 @@ def force_patch_the_trace it "doesn't change the build.trace" do force_patch_the_trace - expect(build.reload.trace).to eq 'BUILD TRACE appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' end end end @@ -368,7 +370,7 @@ def force_patch_the_trace it 'changes the build.trace' do patch_the_trace - expect(build.reload.trace).to eq 'BUILD TRACE appended appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended appended' end context 'when Runner makes a force-patch' do @@ -377,7 +379,7 @@ def force_patch_the_trace it "doesn't change the build.trace" do force_patch_the_trace - expect(build.reload.trace).to eq 'BUILD TRACE appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' end end end @@ -403,7 +405,7 @@ def force_patch_the_trace it 'gets correct response' do expect(response.status).to eq 202 - expect(build.reload.trace).to eq 'BUILD TRACE appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Build-Status' end