@@ -24,6 +24,7 @@ use Osstest;
use Osstest::TestSupport;
use Osstest::Executive;
use Data::Dumper;
+use Carp;
BEGIN {
use Exporter ();
@@ -461,4 +462,214 @@ sub jobdb_db_glob ($$) { #method
sub can_anoint ($) { return 1; }
+sub jobdb_host_update_lifecycle_info ($$$) { #method
+ my ($mo, $ho, $mode) = @_;
+ # $mode is sigil
+ # selectprep 1 @ called several times, from selecthost, $isprep
+ # wiped 2 n/a called (usually) once, when install succeeds
+ # select 1 + called several times, from selecthost, !$isprep
+ # final 1 none called (hopefully) once, from capture-logs
+ # (once means within this job)
+ # Notes:
+ # 1 causes a new row to be added to the host lifecycle
+ # subject to sigil check
+ # 2 removes other rows from the host lifecycle
+ #
+ # Where "sigil" above is nonempty: we look at the runvar. If the
+ # runvar's final sigil is the same, we conclude that the needed
+ # work has already been done: ie, we do not need to add a row to
+ # the table nor do we necessarily need to update the runvar.
+ #
+ # If the sigil is not right, we replace the runvar with a history
+ # string derived from the lifecycle table, and (if appropriate)
+ # add a row.
+ #
+ # In principle it might be useful to update the runvar every time
+ # because the lifecycle table might have gained rows from other
+ # jobs, but we would like to avoid burdening select with more
+ # pratting about. So we leave that for capture-logs and
+ # reuse/final, ie when newsigil is none.
+ #
+ # The runvar is
+ # <ident>_lifecycle
+ # and contains space-separated entries
+ # [+@][<flight>.][<job>]:<stepno>[,<ident>]
+ # [+@]<stepno> same flight, job, ident; new stepno
+ # "["<omitted-count>"]" ie literal [ ]
+ # [+]?<taskid> task not within a flight/job
+ # where omitted [,<ident>] means "host";
+ # omitted <flight> and <job> mean this runvar's flight and/or job;
+ # and then at then end most one of the sigils
+ # @ last call was selectprep
+ # + last call was select
+ # <none> last call was final
+ # items with no such sigil don't appear in build jobs
+ # and instead appear as [<omitted-count>] eg "[4]"
+
+ return if $ho->{Host}; # This host *has* a host - ie, nested
+
+ my $ttaskid = findtask();
+ my $hostname = $ho->{Name};
+ my $tident = $ho->{Ident};
+ my $tstepno = $mo->current_stepno();
+
+ if ($mode eq 'wiped') {
+ db_retry($flight, [qw(running)], $dbh_tests,[], sub {
+ $dbh_tests->do(<<END, {}, $hostname);
+ DELETE FROM host_lifecycle h
+ WHERE hostname=?
+ AND NOT EXISTS(
+ SELECT 1
+ FROM tasks t
+ WHERE t.live
+ AND t.taskid = h.taskid
+ );
+END
+ });
+ logm("host lifecycle: $hostname: wiped, cleared out old info");
+ return;
+ }
+
+ my $newsigil =
+ $mode eq 'selectprep' ? '@' :
+ $mode eq 'select' ? '+' :
+ $mode eq 'final' ? '' :
+ confess "$mode ?";
+
+ my $scanq = $dbh_tests->prepare(<<END);
+ SELECT h.flight, h.job, h.isprep, h.ident, h.stepno,
+ t.live, t.taskid,
+ h2.lcseq later_notprep
+ FROM host_lifecycle h
+ LEFT JOIN host_lifecycle h2
+ ON h2.hostname = h.hostname
+ AND h2.flight = h.flight
+ AND h2.job = h.job
+ AND h2.ident = h.ident
+ AND h2.taskid = h.taskid
+ AND h2.lcseq > h.lcseq
+ AND h.isprep AND NOT h2.isprep
+ LEFT JOIN tasks t
+ ON h.taskid = t.taskid
+ WHERE h.hostname = ?
+ ORDER BY h.lcseq;
+END
+ my $insertq = $dbh_tests->prepare(<<END);
+ INSERT INTO host_lifecycle
+ (hostname, taskid, flight, job, isprep, ident, stepno)
+ VALUES (?, ?, ?, ?, ?, ?, ? )
+END
+
+ my $ojvn = "$ho->{Ident}_lifecycle";
+ my $firstrun;
+
+ if (length $r{$ojvn}) {
+ my ($oldsigil,) = reverse split / /, $r{$ojvn};
+ $oldsigil = '' unless $oldsigil =~ m/^\W$/;
+ return if $newsigil ne '' && $oldsigil eq $newsigil;
+ } else {
+ $firstrun = 1;
+ }
+
+ my @lifecycle;
+ db_retry($dbh_tests,[], sub {
+ my $elided;
+ @lifecycle = ();
+ my %tj_seen;
+ # keys in %tj_seen are [@][<flight>.][<job>] or ?<taskid>
+ $scanq->execute($hostname);
+
+ while (my $o = $scanq->fetchrow_hashref()) {
+ my $olive =
+ # Any job which appeared since we started thinking
+ # about this must have been concurrent with us,
+ # even if it is dead now.
+ (!$firstrun || $o->{live}) &&
+ # If this task is still live, we need to have something
+ # with a live mark, generally all the prep will have
+ # occurred already, so we don't mark the prep as live
+ # if there's a later nonprep step.
+ !$o->{later_notprep};
+
+ my $olivemark = !!$olive && '+';
+ if (defined($flight) && defined($o->{flight}) &&
+ $o->{flight} eq $flight &&
+ $o->{job} eq $job) {
+ # Don't put the + mark on our own entries.
+ $olivemark = '';
+ }
+
+ my $oisprepmark = !!$o->{isprep} && '@';
+ my $omarks = $olivemark.$oisprepmark;
+
+ my $otj = '';
+ if (!defined $o->{flight}) {
+ $otj .= "?$o->{taskid}";
+ } else {
+ $otj .= "$o->{flight}." if $o->{flight} ne $flight;
+ $otj .= $o->{job} if $o->{job} ne $job;
+ }
+ next if $tj_seen{$oisprepmark.$otj}++;
+
+ if (!$omarks && !$olive && defined($o->{flight}) &&
+ $ho->{Shared} &&
+ $ho->{Shared}{Type} =~ m/^build-/ &&
+ !$tj_seen{"\@$otj"} # do not elide use if we showed prep
+ ) {
+ # elide previous, non-concurrent, build jobs
+ if (!$elided) { $elided = [ scalar(@lifecycle), 0]; }
+ $lifecycle[$elided->[0]] = "[".(++$elided->[1])."]";
+ next;
+ }
+
+ my $osuffix = !!(defined($o->{ident}) && $o->{ident} ne 'host')
+ && ",$o->{ident}";
+
+ my ($lastuncompr,) = grep { !m{^\W*\d+$} } reverse @lifecycle;
+ if (defined($lastuncompr) &&
+ $lastuncompr =~ m{^\W*\Q$otj\E:\d+\Q$osuffix\E$}) {
+ push @lifecycle, "$omarks$o->{stepno}";
+ } else {
+ push @lifecycle, "$omarks$otj:$o->{stepno}$osuffix";
+ }
+ }
+ if (defined $flight) {
+ $insertq->execute($hostname, $ttaskid,
+ $flight, $job,
+ ($mode eq 'selectprep')+0,
+ # ^ DBD::Pg doesn't accept perl canonical false for bool!
+ # https://rt.cpan.org/Public/Bug/Display.html?id=133229
+ $tident, $tstepno);
+ } else {
+ $insertq->execute($hostname, $ttaskid,
+ undef,undef,
+ undef,
+ undef,undef);
+ }
+ });
+
+ if (defined $flight) {
+ push @lifecycle, $newsigil if length $newsigil;
+ store_runvar($ojvn, "@lifecycle");
+ }
+}
+
+sub current_stepno ($) { #method
+ my ($jd) = @_;
+ my $testid = $ENV{OSSTEST_TESTID} // return undef;
+ my $checkq = $dbh_tests->prepare(<<END);
+ SELECT stepno
+ FROM steps
+ WHERE flight=?
+ AND job=?
+ AND testid=?
+END
+ my $stepno;
+ db_retry($flight,[qw(running)], $dbh_tests,[],sub {
+ $checkq->execute($flight,$job,$testid);
+ ($stepno) = $checkq->fetchrow_array();
+ });
+ return $stepno;
+}
+
1;
@@ -137,4 +137,6 @@ sub jobdb_db_glob ($) { #method
sub can_anoint ($) { return 0; }
+sub jobdb_host_update_lifecycle_info { } #method
+
1;
@@ -88,6 +88,7 @@ BEGIN {
serial_fetch_logs set_host_property modify_host_flag
propname_massage propname_check
hostprop_putative_record hostflag_putative_record
+ host_update_lifecycle_info
get_stashed open_unique_stashfile compress_stashed
dir_identify_vcs
@@ -174,6 +175,7 @@ our @accessible_runvar_pats =
host_console *_host_console
host_hostflagadjust *_host_hostflagadjust
host_hostflags *_host_hostflags
+ host_lifecycle *_host_lifecycle
host_linux_boot_append *_host_linux_boot_append
host_ip *_host_ip
host_power_install *_host_power_install
@@ -3166,6 +3168,16 @@ sub sha256file ($;$) {
return $truncate ? substr($digest, 0, $truncate) : $digest;
}
+sub host_update_lifecycle_info ($$) {
+ my ($ho, $mode) = @_;
+ # $mode is
+ # selectprep called several times, from selecthost, $isprep
+ # wiped called once, when install succeeds
+ # select called several times, from selecthost, !$isprep
+ # final called hopefully once, from capture-logs
+ $mjobdb->jobdb_host_update_lifecycle_info($ho, $mode)
+}
+
sub host_shared_mark_ready($$;$$) {
my ($ho,$sharetype, $oldstate, $newstate) = @_;
new file mode 100644
@@ -0,0 +1,43 @@
+-- ##OSSTEST## 012 Preparatory
+--
+-- Records the jobs which have touched a host, for host sharing/reuse
+-- The information here is ephemeral - it is cleared when a host is
+-- reinitialised. The information is persisted by being copied
+-- into a runvar for each job.
+
+CREATE SEQUENCE host_lifecycle_lcseq_seq
+ NO CYCLE;
+
+CREATE TABLE host_lifecycle (
+ hostname TEXT NOT NULL,
+ lcseq INTEGER NOT NULL DEFAULT nextval('host_lifecycle_lcseq_seq'),
+ taskid INTEGER NOT NULL, -- no constraint, tasks can get deleted
+ flight INTEGER,
+ job TEXT,
+ stepno INTEGER,
+ ident TEXT,
+ isprep BOOLEAN,
+ PRIMARY KEY (hostname, lcseq),
+-- restype TEXT GENERATED ALWAYS AS ('host'),
+-- FOREIGN KEY (restype,hostname) REFERENCES resources(restype, resname),
+-- ^ those two omitted because not supported until pgsql 13
+-- FOREIGN KEY (flight, job) REFERENCES jobs(flight, job),
+-- ^ omitted because the next constraint implies it
+ FOREIGN KEY (flight, job, stepno) REFERENCES steps(flight, job, stepno),
+ CHECK ((
+ flight IS NOT NULL AND
+ job IS NOT NULL AND
+ stepno IS NOT NULL AND
+ ident IS NOT NULL AND
+ isprep IS NOT NULL
+ ) OR (
+ flight IS NULL AND
+ job IS NULL AND
+ stepno IS NULL AND
+ ident IS NULL AND
+ isprep IS NULL
+ ))
+);
+
+ALTER SEQUENCE host_lifecycle_lcseq_seq
+ OWNED BY host_lifecycle.lcseq;