From: Francois Gouget Subject: [15/25] testbot: Share code to access the latest reference WineTest reports. Message-Id: Date: Tue, 14 Jan 2020 16:42:29 +0100 (CET) In-Reply-To: References: Also take the latest WineTest reports snapshot when the corresponding task completes rather than when the build task they depend on completes. This way the snapshot is taken closest to when the reference reports are needed. --- testbot/bin/UpdateTaskLogs | 12 ++-- testbot/bin/WineRunBuild.pl | 31 --------- testbot/bin/WineRunTask.pl | 34 ++++----- testbot/bin/WineRunWineTest.pl | 65 +++-------------- testbot/lib/WineTestBot/LogUtils.pm | 104 +++++++++++++++++++++++++++- testbot/lib/WineTestBot/Tasks.pm | 30 ++++++++ 6 files changed, 160 insertions(+), 116 deletions(-) diff --git a/testbot/bin/UpdateTaskLogs b/testbot/bin/UpdateTaskLogs index 758c5a7e6..e0af09768 100755 --- a/testbot/bin/UpdateTaskLogs +++ b/testbot/bin/UpdateTaskLogs @@ -226,15 +226,11 @@ sub DoUpdateLatestReport($$$) # Add the reference report to latest/ Debug("latest: Adding $RefReportName from ". Path2TaskKey($SrcReportPath) ."\n"); - foreach my $Suffix ("", ".err") + my $ErrMessages = UpdateLatestReport($Task, $ReportName, $SrcReportPath); + foreach my $ErrMessage (@$ErrMessages) { - unlink("$LatestReportPath$Suffix"); - if (-f "$SrcReportPath$Suffix" and - !link("$SrcReportPath$Suffix", "$LatestReportPath$Suffix")) - { - Error "Could not replace 'latest/$RefReportName$Suffix': $!\n"; - $Rc = 1; - } + Error "$ErrMessage\n"; + $Rc = 1; } $LatestReports{$RefReportName} = $LatestReportPath; } diff --git a/testbot/bin/WineRunBuild.pl b/testbot/bin/WineRunBuild.pl index 5bc3815c5..afbc5ec78 100755 --- a/testbot/bin/WineRunBuild.pl +++ b/testbot/bin/WineRunBuild.pl @@ -496,37 +496,6 @@ Debug(Elapsed($Start), " Disconnecting\n"); $TA->Disconnect(); -# -# Grab a copy of the reference logs -# - -# Note that this may be a bit inaccurate right after a Wine commit. -# See WineSendLog.pl for more details. -my $LatestDir = "$DataDir/latest"; -foreach my $TestStep (@{$Job->Steps->GetItems()}) -{ - if (($TestStep->PreviousNo || 0) == $Step->No and - $TestStep->FileType =~ /^exe/) - { - foreach my $TestTask (@{$TestStep->Tasks->GetItems()}) - { - my $RefReport = $TestTask->GetRefReportName($TestStep->FileType .".report"); - for my $Suffix ("", ".err") - { - if (-f "$LatestDir/$RefReport$Suffix") - { - unlink "$StepDir/$RefReport$Suffix"; - if (!link "$LatestDir/$RefReport$Suffix", "$StepDir/$RefReport$Suffix") - { - Error "Could not link '$RefReport$Suffix': $!\n"; - } - } - } - } - } -} - - # # Wrap up # diff --git a/testbot/bin/WineRunTask.pl b/testbot/bin/WineRunTask.pl index 645798bbb..ee910cdf0 100755 --- a/testbot/bin/WineRunTask.pl +++ b/testbot/bin/WineRunTask.pl @@ -45,7 +45,6 @@ use WineTestBot::Engine::Notify; use WineTestBot::Jobs; use WineTestBot::Log; use WineTestBot::LogUtils; -use WineTestBot::Missions; use WineTestBot::Utils; use WineTestBot::VMs; @@ -195,7 +194,6 @@ my $OldUMask = umask(002); my $TaskDir = $Task->CreateDir(); umask($OldUMask); my $VM = $Task->VM; -my $RptFileName = $Step->FileType .".report"; my $Start = Time(); @@ -224,6 +222,8 @@ sub LogTaskError($) umask($OldUMask); } +my $ReportNames; + sub WrapUpAndExit($;$$$$) { my ($Status, $TestFailures, $Retry, $TimedOut, $Reason) = @_; @@ -299,22 +299,11 @@ sub WrapUpAndExit($;$$$$) $VM->Save(); } + # Update the 'latest/' reference WineTest results if ($Step->Type eq 'suite' and $Status eq 'completed' and !$TimedOut) { - # Keep the old report if the new one is missing - if (-f "$TaskDir/$RptFileName" and !-z "$TaskDir/$RptFileName") - { - # Update the VM's reference WineTest results for WineSendLog.pl - my $RefReport = "$DataDir/latest/". $Task->GetRefReportName($RptFileName); - unlink($RefReport); - link("$TaskDir/$RptFileName", $RefReport); - - unlink("$RefReport.err"); - if (-f "$TaskDir/$RptFileName.err" and !-z "$TaskDir/$RptFileName.err") - { - link("$TaskDir/$RptFileName.err", "$RefReport.err"); - } - } + my $ErrMessages = UpdateLatestReports($Task, $ReportNames); + Error("$_\n") for (@$ErrMessages); } my $Result = $VM->Name .": ". $VM->Status ." Status: $Status Failures: ". (defined $TestFailures ? $TestFailures : "unset"); @@ -420,12 +409,11 @@ if ($Step->FileType ne "exe32" and $Step->FileType ne "exe64") FatalError("Unexpected file type '". $Step->FileType ."' found for ". $Step->Type ." step\n"); } -my ($ErrMessage, $Missions) = ParseMissionStatement($Task->Missions); +(my $ErrMessage, $ReportNames, my $TaskMissions) = $Task->GetReportNames(); FatalError "$ErrMessage\n" if (defined $ErrMessage); -FatalError "Empty mission statement\n" if (!@$Missions); -FatalError "Cannot specify missions for multiple tasks\n" if (@$Missions > 1); -FatalError "Cannot specify multiple missions\n" if (@{$Missions->[0]->{Missions}} > 1); -my $Mission = $Missions->[0]->{Missions}->[0]; +FatalError "Cannot specify multiple missions\n" if (@{$TaskMissions->{Missions}} > 1); +my $Mission = $TaskMissions->{Missions}->[0]; +my $RptFileName = $ReportNames->[0]; # @@ -567,6 +555,10 @@ if ($TA->GetFile($RptFileName, "$TaskDir/$RptFileName")) } else { + # Grab a copy of the reference report + my $ErrMessages = SnapshotLatestReport($Task, $RptFileName); + LogTaskError("$_\n") for (@$ErrMessages); + # $LogInfo->{Failures} can legitimately be undefined in case of a timeout $TaskFailures += $LogInfo->{Failures} || 0; if (@{$LogInfo->{Extra}} and open(my $Log, ">", "$TaskDir/$RptFileName.err")) diff --git a/testbot/bin/WineRunWineTest.pl b/testbot/bin/WineRunWineTest.pl index ea13ca02d..46e85ec47 100755 --- a/testbot/bin/WineRunWineTest.pl +++ b/testbot/bin/WineRunWineTest.pl @@ -43,7 +43,6 @@ $Name0 =~ s+^.*/++; use WineTestBot::Config; use WineTestBot::Engine::Notify; use WineTestBot::Jobs; -use WineTestBot::Missions; use WineTestBot::PatchUtils; use WineTestBot::Log; use WineTestBot::LogUtils; @@ -218,7 +217,7 @@ sub LogTaskError($) } } -my $TaskMissions; +my $ReportNames; sub WrapUpAndExit($;$$$$) { @@ -295,26 +294,11 @@ sub WrapUpAndExit($;$$$$) $VM->Save(); } + # Update the 'latest/' reference WineTest results if ($Step->Type eq 'suite' and $Status eq 'completed' and !$TimedOut) { - foreach my $Mission (@{$TaskMissions->{Missions}}) - { - # Keep the old report if the new one is missing - my $RptFileName = GetMissionBaseName($Mission) .".report"; - if (-f "$TaskDir/$RptFileName" and !-z "$TaskDir/$RptFileName") - { - # Update the VM's reference WineTest results for WineSendLog.pl - my $RefReport = "$DataDir/latest/". $Task->VM->Name ."_$RptFileName"; - unlink($RefReport); - link("$TaskDir/$RptFileName", $RefReport); - - unlink("$RefReport.err"); - if (-f "$TaskDir/$RptFileName.err" and !-z "$TaskDir/$RptFileName.err") - { - link("$TaskDir/$RptFileName.err", "$RefReport.err"); - } - } - } + my $ErrMessages = UpdateLatestReports($Task, $ReportNames); + Error("$_\n") for (@$ErrMessages); } my $Result = $VM->Name .": ". $VM->Status ." Status: $Status Failures: ". (defined $TestFailures ? $TestFailures : "unset"); @@ -421,11 +405,8 @@ if (($Step->Type eq "suite" and $Step->FileType ne "none") or FatalError("Unexpected file type '". $Step->FileType ."' found for ". $Step->Type ." step\n"); } -my ($ErrMessage, $Missions) = ParseMissionStatement($Task->Missions); +(my $ErrMessage, $ReportNames, my $_TaskMissions) = $Task->GetReportNames(); FatalError "$ErrMessage\n" if (defined $ErrMessage); -FatalError "Empty mission statement\n" if (!@$Missions); -FatalError "Cannot specify missions for multiple tasks\n" if (@$Missions > 1); -$TaskMissions = $Missions->[0]; # @@ -570,9 +551,8 @@ elsif (!defined $TAError) # Grab the test reports if any # -foreach my $Mission (@{$TaskMissions->{Missions}}) +foreach my $RptFileName (@$ReportNames) { - my $RptFileName = GetMissionBaseName($Mission) .".report"; Debug(Elapsed($Start), " Retrieving '$RptFileName'\n"); if ($TA->GetFile($RptFileName, "$TaskDir/$RptFileName")) { @@ -589,6 +569,10 @@ foreach my $Mission (@{$TaskMissions->{Missions}}) } else { + # Grab a copy of the reference report + my $ErrMessages = SnapshotLatestReport($Task, $RptFileName); + LogTaskError("$_\n") for (@$ErrMessages); + # $LogInfo->{Failures} can legitimately be undefined in case of a timeout $TaskFailures += $LogInfo->{Failures} || 0; if (@{$LogInfo->{Extra}} and open(my $Log, ">", "$TaskDir/$RptFileName.err")) @@ -617,35 +601,6 @@ Debug(Elapsed($Start), " Disconnecting\n"); $TA->Disconnect(); -# -# Grab a copy of the reference logs -# - -# Note that this may be a bit inaccurate right after a Wine commit. -# See WineSendLog.pl for more details. -if ($NewStatus eq 'completed') -{ - my $LatestDir = "$DataDir/latest"; - my $StepDir = $Step->GetDir(); - foreach my $Mission (@{$TaskMissions->{Missions}}) - { - my $RptFileName = GetMissionBaseName($Mission) .".report"; - my $RefReport = $Task->GetRefReportName($RptFileName); - for my $Suffix ("", ".err") - { - if (-f "$LatestDir/$RefReport$Suffix") - { - unlink "$StepDir/$RefReport$Suffix"; - if (!link "$LatestDir/$RefReport$Suffix", "$StepDir/$RefReport$Suffix") - { - Error "Could not link '$RefReport$Suffix': $!\n"; - } - } - } - } -} - - # # Wrap up # diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm index 3e93c95c7..764330281 100644 --- a/testbot/lib/WineTestBot/LogUtils.pm +++ b/testbot/lib/WineTestBot/LogUtils.pm @@ -29,7 +29,8 @@ WineTestBot::LogUtils - Provides functions to parse task logs use Exporter 'import'; our @EXPORT = qw(GetLogFileNames GetLogLabel GetLogErrors TagNewErrors GetLogLineCategory GetReportLineCategory - ParseTaskLog ParseWineTestReport); + ParseTaskLog ParseWineTestReport + SnapshotLatestReport UpdateLatestReport UpdateLatestReports); use Algorithm::Diff; use File::Basename; @@ -868,6 +869,11 @@ sub GetLogErrors($) return $LogInfo; } + +# +# New error detection +# + sub _DumpDiff($$) { my ($Label, $Diff) = @_; @@ -1025,4 +1031,100 @@ sub TagNewErrors($$) } } + +# +# Reference report management +# + +=pod +=over 12 + +=item C + +Takes a snapshot of the reference WineTest results for the specified Task. + +The reference report is used to identify new failures, even long after the task has been +run (and the reference report replaced by a newer version). + +Note also that comparing reports in this way may be a bit inaccurate right +after a Wine commit due to delays in getting new WineTest results, etc. +See WineSendLog.pl for more details. + +=back +=cut + +sub SnapshotLatestReport($$) +{ + my ($Task, $ReportName) = @_; + + my @ErrMessages; + my $TaskDir = $Task->GetDir(); + my $RefReportName = $Task->GetRefReportName($ReportName); + foreach my $Suffix ("", ".err") + { + next if (!-f "$DataDir/latest/$RefReportName$Suffix"); + + # FIXME: The reference reports are stored at the step level! + if (!link("$DataDir/latest/$RefReportName$Suffix", + "$TaskDir/../$RefReportName$Suffix")) + { + push @ErrMessages, "Could not create the '../$RefReportName$Suffix' link: $!"; + } + } + + return \@ErrMessages; +} + +sub UpdateLatestReport($$$) +{ + my ($Task, $ReportName, $SrcReportPath) = @_; + my @ErrMessages; + + my $RefReportName = $Task->GetRefReportName($ReportName); + foreach my $Suffix ("", ".err") + { + # Add the new reference file even if it is empty. + next if (!-f "$SrcReportPath$Suffix"); + + unlink "$DataDir/latest/$RefReportName$Suffix"; + if (!link("$SrcReportPath$Suffix", + "$DataDir/latest/$RefReportName$Suffix")) + { + push @ErrMessages, "Could not create the '$RefReportName$Suffix' link: $!"; + } + } + + return \@ErrMessages; +} + +=pod +=over 12 + +=item C + +Adds the Task's WineTest results to the set of reference reports. + +The reference reports will then be used to detect new failures in the other +tasks. + +This must be called after SnapshotLatestReport() otherwise a WineTest task +would compare its results to itself. + +=back +=cut + +sub UpdateLatestReports($$) +{ + my ($Task, $ReportNames) = @_; + + my @ErrMessages; + my $TaskDir = $Task->GetDir(); + foreach my $ReportName (@$ReportNames) + { + next if (!-f "$TaskDir/$ReportName" or -z _); + push @ErrMessages, @{UpdateLatestReport($Task, $ReportName, "$TaskDir/$ReportName")}; + } + return \@ErrMessages; +} + 1; diff --git a/testbot/lib/WineTestBot/Tasks.pm b/testbot/lib/WineTestBot/Tasks.pm index 0c26904e1..1a46f0d57 100644 --- a/testbot/lib/WineTestBot/Tasks.pm +++ b/testbot/lib/WineTestBot/Tasks.pm @@ -70,6 +70,7 @@ our @ISA = qw(WineTestBot::WineTestBotItem); use File::Path; use ObjectModel::BackEnd; use WineTestBot::Config; +use WineTestBot::Missions; sub InitializeNew($$) @@ -108,6 +109,35 @@ sub RmTree($) rmtree($Dir); } +sub GetReportNames($) +{ + my ($self) = @_; + + my ($ErrMessage, $Missions) = ParseMissionStatement($self->Missions); + return ($ErrMessage, undef, undef) if (defined $ErrMessage); + if (!@$Missions) + { + my @TaskKey = $self->GetMasterKey(); + return ("Task @TaskKey has no mission", undef, undef); + } + if (@$Missions > 1) + { + my @TaskKey = $self->GetMasterKey(); + return ("Task @TaskKey should not have missions for multiple tasks\n", undef); + } + my $TaskMissions = $Missions->[0]; + + my @ReportNames; + foreach my $Mission (@{$TaskMissions->{Missions}}) + { + if ($Mission->{test} ne "build") + { + push @ReportNames, GetMissionBaseName($Mission) .".report"; + } + } + return (undef, \@ReportNames, $TaskMissions); +} + sub GetRefReportName($$) { my ($self, $ReportName) = @_; -- 2.20.1