sync-all 23.7 KB
Newer Older
1 2 3 4 5
#!/usr/bin/perl -w

use strict;
use Cwd;

Ian Lynagh's avatar
Ian Lynagh committed
6 7
$| = 1; # autoflush stdout after each print, to avoid output after die

Simon Marlow's avatar
Simon Marlow committed
8
my $defaultrepo;
Simon Marlow's avatar
Simon Marlow committed
9
my @packages;
10
my $verbose = 2;
11
my $try_to_resume = 0;
Simon Marlow's avatar
Simon Marlow committed
12
my $ignore_failure = 0;
13
my $checked_out_flag = 0; # NOT the opposite of bare_flag (describes remote repo state)
14
my $get_mode;
15
my $bare_flag = ""; # NOT the opposite of checked_out_flag (describes local repo state)
16 17

my %tags;
Simon Marlow's avatar
Simon Marlow committed
18 19 20

# Figure out where to get the other repositories from.
sub getrepo {
Simon Marlow's avatar
Simon Marlow committed
21 22 23 24 25 26 27 28
    my $repo;

    if (defined($defaultrepo)) {
        $repo = $defaultrepo;
        chomp $repo;
    } else {
        # Figure out where to get the other repositories from,
        # based on where this GHC repo came from.
29 30 31 32 33 34 35 36
        my $git_dir = $bare_flag ? "--git-dir=ghc.git" : "";
        my $branch  = `git $git_dir branch | grep "\* " | sed "s/^\* //"`; chomp $branch;
        my $remote  = `git $git_dir config branch.$branch.remote`;         chomp $remote;
        if ($remote eq "") {
            # remotes are not mandatory for branches (e.g. not recorded by default for bare repos)
            $remote = "origin";
        }
        $repo       = `git $git_dir config remote.$remote.url`;            chomp $repo;
Simon Marlow's avatar
Simon Marlow committed
37
    }
Simon Marlow's avatar
Simon Marlow committed
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61

    my $repo_base;
    my $checked_out_tree;

    if ($repo =~ /^...*:/) {
        # HTTP or SSH
        # Above regex says "at least two chars before the :", to avoid
        # catching Win32 drives ("C:\").
        $repo_base = $repo;

        # --checked-out is needed if you want to use a checked-out repo
        # over SSH or HTTP
        if ($checked_out_flag) {
            $checked_out_tree = 1;
        } else {
            $checked_out_tree = 0;
        }

        # Don't drop the last part of the path if specified with -r, as
        # it expects repos of the form:
        #
        #   http://darcs.haskell.org
        #
        # rather than
daniel.is.fischer's avatar
daniel.is.fischer committed
62
        #
Simon Marlow's avatar
Simon Marlow committed
63 64 65 66 67 68 69
        #   http://darcs.haskell.org/ghc
        #
        if (!$defaultrepo) {
            $repo_base =~ s#/[^/]+/?$##;
        }
    }
    elsif ($repo =~ /^\/|\.\.\/|.:(\/|\\)/) {
70
        # Local filesystem, either absolute (C:/ or /) or relative (../) path
Simon Marlow's avatar
Simon Marlow committed
71
        $repo_base = $repo;
72 73 74 75 76 77 78 79 80 81 82
        if (-f "$repo/HEAD") {
            # assume a local mirror:
            $checked_out_tree = 0;
            $repo_base =~ s#/[^/]+/?$##;
        } elsif (-d "$repo/ghc.git") {
            # assume a local mirror:
            $checked_out_tree = 0;
        } else {
            # assume a checked-out tree:
            $checked_out_tree = 1;
        }
Simon Marlow's avatar
Simon Marlow committed
83 84 85 86 87 88 89 90 91 92 93 94
    }
    else {
        die "Couldn't work out repo";
    }

    return $repo_base, $checked_out_tree;
}

sub parsePackages {
    my @repos;
    my $lineNum;

95 96 97
    open IN, "< packages.conf"
        or open IN, "< packages" # clashes with packages directory when using --bare
        or die "Can't open packages file (or packages.conf)";
Simon Marlow's avatar
Simon Marlow committed
98 99 100 101 102 103 104 105
    @repos = <IN>;
    close IN;

    @packages = ();
    $lineNum = 0;
    foreach (@repos) {
        chomp;
        $lineNum++;
106
        if (/^([^# ]+) +([^ ]+) +([^ ]+) +([^ ]+)$/) {
Simon Marlow's avatar
Simon Marlow committed
107 108 109 110 111 112 113 114 115 116 117 118
            my %line;
            $line{"localpath"}  = $1;
            $line{"tag"}        = $2;
            $line{"remotepath"} = $3;
            $line{"vcs"}        = $4;
            push @packages, \%line;
        }
        elsif (! /^(#.*)?$/) {
            die "Bad content on line $lineNum of packages file: $_";
        }
    }
}
119 120 121 122 123 124 125 126 127 128 129 130 131

sub message {
    if ($verbose >= 2) {
        print "@_\n";
    }
}

sub warning {
    if ($verbose >= 1) {
        print "warning: @_\n";
    }
}

tibbe's avatar
tibbe committed
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
sub gitNewWorkdir {
    my $dir = shift;
    my $target = shift;
    my $target_dir = "$target/$dir";
    my $pwd;

    if ($dir eq '.') {
        message "== running git-new-workdir . $target_dir @_";
    } else {
        message "== $dir: running git-new-workdir . $target_dir @_";
        $pwd = getcwd();
        chdir($dir);
    }

    system ("git-new-workdir", ".", $target_dir, @_) == 0
        or $ignore_failure
        or die "git-new-workdir failed: $?";

    if ($dir ne '.') {
        chdir($pwd);
    }
}

155
sub scm {
Simon Marlow's avatar
Simon Marlow committed
156
    my $dir = shift;
157
    my $scm = shift;
Simon Marlow's avatar
Simon Marlow committed
158 159 160 161 162 163 164 165 166 167
    my $pwd;

    if ($dir eq '.') {
        message "== running $scm @_";
    } else {
        message "== $dir: running $scm @_";
        $pwd = getcwd();
        chdir($dir);
    }

168 169 170
    system ($scm, @_) == 0
        or $ignore_failure
        or die "$scm failed: $?";
Simon Marlow's avatar
Simon Marlow committed
171 172 173 174

    if ($dir ne '.') {
        chdir($pwd);
    }
175 176 177 178
}

sub scmall {
    my $command = shift;
daniel.is.fischer's avatar
daniel.is.fischer committed
179

180 181 182 183
    my $localpath;
    my $tag;
    my $remotepath;
    my $scm;
Simon Marlow's avatar
Simon Marlow committed
184
    my $line;
185 186
    my $branch_name;
    my $subcommand;
187 188 189 190

    my $path;
    my $wd_before = getcwd;

Simon Marlow's avatar
Simon Marlow committed
191
    my $pwd;
192
    my @args;
Simon Marlow's avatar
Simon Marlow committed
193

194 195 196 197
    my $started;
    my $doing;
    my $start_repo;

Simon Marlow's avatar
Simon Marlow committed
198 199
    my ($repo_base, $checked_out_tree) = getrepo();

200
    my $is_github_repo = $repo_base =~ m/(git@|git:\/\/|https:\/\/)github.com/;
201

Simon Marlow's avatar
Simon Marlow committed
202 203
    parsePackages;

204 205
    @args = ();

206
    if ($command =~ /^remote$/) {
207 208
        while (@_ > 0 && $_[0] =~ /^-/) {
            push(@args,shift);
209
        }
210
        if (@_ < 1) { help(1); }
211
        $subcommand = shift;
212 213 214 215
        if ($subcommand ne 'add' &&
            $subcommand ne 'rm' &&
            $subcommand ne 'set-branches' &&
            $subcommand ne 'set-url') {
216
            help(1);
217
        }
218 219 220 221
        while (@_ > 0 && $_[0] =~ /^-/) {
            push(@args,shift);
        }
        if (($subcommand eq 'add' || $subcommand eq 'rm') && @_ < 1) {
222
            help(1);
223 224 225 226 227
        } elsif (@_ < 1) { # set-url
            $branch_name = 'origin';
        } else {
            $branch_name = shift;
        }
Ian Lynagh's avatar
Ian Lynagh committed
228
    } elsif ($command eq 'new') {
229 230 231 232 233
        if (@_ < 1) {
            $branch_name = 'origin';
        } else {
            $branch_name = shift;
        }
234 235
    }

236 237
    push(@args, @_);

238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
    # $doing is a good enough approximation to what we are doing that
    # we can use it to check that --resume is resuming the right command
    $doing = join(" ", ($command, @args));
    $started = 1;
    if ($try_to_resume && -f "resume") {
        my $what;
        open RESUME, "< resume"
            or die "Can't open resume file";
        $start_repo = <RESUME>;
        chomp $start_repo;
        $what = <RESUME>;
        chomp $what;
        close RESUME;
        if ($what eq $doing) {
            $started = 0;
        }
    }

Simon Marlow's avatar
Simon Marlow committed
256
    for $line (@packages) {
Ian Lynagh's avatar
Ian Lynagh committed
257 258
        $tag        = $$line{"tag"};
        $scm        = $$line{"vcs"};
259 260 261
        # Use the "remote" structure for bare git repositories
        $localpath  = ($bare_flag && $scm eq "git") ?
                      $$line{"remotepath"} : $$line{"localpath"};
262
        $remotepath = ($checked_out_tree) ?
263
                      $$line{"localpath"}  : $$line{"remotepath"};
264

265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
        if (!$started) {
            if ($start_repo eq $localpath) {
                $started = 1;
            }
            else {
                next;
            }
        }

        open RESUME, "> resume.tmp";
        print RESUME "$localpath\n";
        print RESUME "$doing\n";
        close RESUME;
        rename "resume.tmp", "resume";

Ian Lynagh's avatar
Ian Lynagh committed
280 281
        # Check the SCM is OK as early as possible
        die "Unknown SCM: $scm" if (($scm ne "darcs") and ($scm ne "git"));
282

Ian Lynagh's avatar
Ian Lynagh committed
283
        # We can't create directories on GitHub, so we translate
284
        # "packages/foo" into "package-foo".
Ian Lynagh's avatar
Ian Lynagh committed
285 286 287
        if ($is_github_repo) {
            $remotepath =~ s/\//-/;
        }
288

289 290
        # Construct the path for this package in the repo we pulled from
        $path = "$repo_base/$remotepath";
Simon Marlow's avatar
Simon Marlow committed
291

Ian Lynagh's avatar
Ian Lynagh committed
292 293 294 295
        if ($command =~ /^(?:g|ge|get)$/) {
            # Skip any repositories we have not included the tag for
            if (not defined($tags{$tag})) {
                $tags{$tag} = 0;
296
            }
Ian Lynagh's avatar
Ian Lynagh committed
297 298
            if ($tags{$tag} == 0) {
                next;
299
            }
daniel.is.fischer's avatar
daniel.is.fischer committed
300

Ian Lynagh's avatar
Ian Lynagh committed
301 302 303
            if (-d $localpath) {
                warning("$localpath already present; omitting")
                    if $localpath ne ".";
304 305 306
                if ($scm eq "git") {
                    scm ($localpath, $scm, "config", "core.ignorecase", "true");
                }
Ian Lynagh's avatar
Ian Lynagh committed
307
                next;
308
            }
309

Ian Lynagh's avatar
Ian Lynagh committed
310 311
            # Note that we use "." as the path, as $localpath
            # doesn't exist yet.
Ian Lynagh's avatar
Ian Lynagh committed
312
            if ($scm eq "darcs") {
313 314 315 316 317
                # The first time round the loop, default the get-mode
                if (not defined($get_mode)) {
                    warning("adding --partial, to override use --complete");
                    $get_mode = "--partial";
                }
Ian Lynagh's avatar
Ian Lynagh committed
318
                scm (".", $scm, "get", $get_mode, $path, $localpath, @args);
tibbe's avatar
tibbe committed
319
            }
Ian Lynagh's avatar
Ian Lynagh committed
320
            else {
Ian Lynagh's avatar
Ian Lynagh committed
321 322 323
                my @argsWithBare = @args;
                push @argsWithBare, $bare_flag if $bare_flag;
                scm (".", $scm, "clone", $path, $localpath, @argsWithBare);
324
                scm ($localpath, $scm, "config", "core.ignorecase", "true");
Ian Lynagh's avatar
Ian Lynagh committed
325
            }
Ian Lynagh's avatar
Ian Lynagh committed
326 327 328
            next;
        }

329 330 331 332
        my $darcs_repo_present = 1 if -d "$localpath/_darcs";
        my $git_repo_present = 1 if -d "$localpath/.git" || ($bare_flag && -d "$localpath");
        if ($darcs_repo_present) {
            if ($git_repo_present) {
Ian Lynagh's avatar
Ian Lynagh committed
333
                die "Found both _darcs and .git in $localpath";
Ian Lynagh's avatar
Ian Lynagh committed
334
            }
kili's avatar
kili committed
335
            $scm = "darcs";
336
        } elsif ($git_repo_present) {
kili's avatar
kili committed
337 338 339 340 341 342
            $scm = "git";
        } elsif ($tag eq "") {
            die "Required repo $localpath is missing";
        } else {
             message "== $localpath repo not present; skipping";
             next;
Ian Lynagh's avatar
Ian Lynagh committed
343 344 345 346
        }

        # Work out the arguments we should give to the SCM
        if ($command =~ /^(?:w|wh|wha|what|whats|whatsn|whatsne|whatsnew|status)$/) {
Ian Lynagh's avatar
Ian Lynagh committed
347 348 349 350 351 352 353 354 355 356
            if ($scm eq "darcs") {
                $command = "whatsnew";
            }
            elsif ($scm eq "git") {
                $command = "status";
            }
            else {
                die "Unknown scm";
            }

Ian Lynagh's avatar
Ian Lynagh committed
357 358
            # Hack around 'darcs whatsnew' failing if there are no changes
            $ignore_failure = 1;
Ian Lynagh's avatar
Ian Lynagh committed
359
            scm ($localpath, $scm, $command, @args);
Ian Lynagh's avatar
Ian Lynagh committed
360 361 362 363
        }
        elsif ($command =~ /^commit$/) {
            # git fails if there is nothing to commit, so ignore failures
            $ignore_failure = 1;
Ian Lynagh's avatar
Ian Lynagh committed
364
            scm ($localpath, $scm, "commit", @args);
Ian Lynagh's avatar
Ian Lynagh committed
365 366
        }
        elsif ($command =~ /^(?:pus|push)$/) {
Ian Lynagh's avatar
Ian Lynagh committed
367
            scm ($localpath, $scm, "push", @args);
Ian Lynagh's avatar
Ian Lynagh committed
368 369
        }
        elsif ($command =~ /^(?:pul|pull)$/) {
Ian Lynagh's avatar
Ian Lynagh committed
370
            scm ($localpath, $scm, "pull", @args);
Ian Lynagh's avatar
Ian Lynagh committed
371
        }
tibbe's avatar
tibbe committed
372 373 374
        elsif ($command =~ /^(?:new-workdir)$/) {
            gitNewWorkdir ($localpath, @args);
        }
Ian Lynagh's avatar
Ian Lynagh committed
375
        elsif ($command =~ /^(?:s|se|sen|send)$/) {
Ian Lynagh's avatar
Ian Lynagh committed
376 377 378 379 380 381 382 383 384 385
            if ($scm eq "darcs") {
                $command = "send";
            }
            elsif ($scm eq "git") {
                $command = "send-email";
            }
            else {
                die "Unknown scm";
            }
            scm ($localpath, $scm, $command, @args);
Ian Lynagh's avatar
Ian Lynagh committed
386 387
        }
        elsif ($command =~ /^fetch$/) {
Ian Lynagh's avatar
Ian Lynagh committed
388
            scm ($localpath, $scm, "fetch", @args);
Ian Lynagh's avatar
Ian Lynagh committed
389 390
        }
        elsif ($command =~ /^new$/) {
Ian Lynagh's avatar
Ian Lynagh committed
391 392
            my @scm_args = ("log", "$branch_name..");
            scm ($localpath, $scm, @scm_args, @args);
Ian Lynagh's avatar
Ian Lynagh committed
393
        }
Simon Marlow's avatar
Simon Marlow committed
394 395 396
        elsif ($command =~ /^log$/) {
            scm ($localpath, $scm, "log", @args);
        }
Ian Lynagh's avatar
Ian Lynagh committed
397
        elsif ($command =~ /^remote$/) {
Ian Lynagh's avatar
Ian Lynagh committed
398
            my @scm_args;
399
            $ignore_failure = 1;
Ian Lynagh's avatar
Ian Lynagh committed
400 401 402 403
            if ($subcommand eq 'add') {
                @scm_args = ("remote", "add", $branch_name, $path);
            } elsif ($subcommand eq 'rm') {
                @scm_args = ("remote", "rm", $branch_name);
404 405
            } elsif ($subcommand eq 'set-branches') {
                @scm_args = ("remote", "set-branches", $branch_name);
Ian Lynagh's avatar
Ian Lynagh committed
406 407 408
            } elsif ($subcommand eq 'set-url') {
                @scm_args = ("remote", "set-url", $branch_name, $path);
            }
Ian Lynagh's avatar
Ian Lynagh committed
409
            scm ($localpath, $scm, @scm_args, @args);
Ian Lynagh's avatar
Ian Lynagh committed
410
        }
Ian Lynagh's avatar
Ian Lynagh committed
411
        elsif ($command =~ /^checkout$/) {
412 413
            # Not all repos are necessarily branched, so ignore failure
            $ignore_failure = 1;
Ian Lynagh's avatar
Ian Lynagh committed
414 415 416
            scm ($localpath, $scm, "checkout", @args)
                unless $scm eq "darcs";
        }
Ian Lynagh's avatar
Ian Lynagh committed
417
        elsif ($command =~ /^grep$/) {
Ian Lynagh's avatar
Ian Lynagh committed
418 419 420 421
            # Hack around 'git grep' failing if there are no matches
            $ignore_failure = 1;
            scm ($localpath, $scm, "grep", @args)
                unless $scm eq "darcs";
Ian Lynagh's avatar
Ian Lynagh committed
422
        }
Ian Lynagh's avatar
Ian Lynagh committed
423 424 425 426
        elsif ($command =~ /^diff$/) {
            scm ($localpath, $scm, "diff", @args)
                unless $scm eq "darcs";
        }
kili's avatar
kili committed
427 428 429 430
        elsif ($command =~ /^clean$/) {
            scm ($localpath, $scm, "clean", @args)
                unless $scm eq "darcs";
        }
Ian Lynagh's avatar
Ian Lynagh committed
431
        elsif ($command =~ /^reset$/) {
Ian Lynagh's avatar
Ian Lynagh committed
432 433
            scm ($localpath, $scm, "reset", @args)
                unless $scm eq "darcs";
Ian Lynagh's avatar
Ian Lynagh committed
434
        }
435 436 437 438
        elsif ($command =~ /^branch$/) {
            scm ($localpath, $scm, "branch", @args)
                unless $scm eq "darcs";
        }
Ian Lynagh's avatar
Ian Lynagh committed
439
        elsif ($command =~ /^config$/) {
Ian Lynagh's avatar
Ian Lynagh committed
440 441
            scm ($localpath, $scm, "config", @args)
                unless $scm eq "darcs";
Ian Lynagh's avatar
Ian Lynagh committed
442
        }
443 444 445 446
        elsif ($command =~ /^repack$/) {
            scm ($localpath, $scm, "repack", @args)
                if $scm eq "git"
        }
447 448 449 450
        elsif ($command =~ /^format-patch$/) {
            scm ($localpath, $scm, "format-patch", @args)
                if $scm eq "git"
        }
Edward Z. Yang's avatar
Edward Z. Yang committed
451 452 453 454
        elsif ($command =~ /^gc$/) {
            scm ($localpath, $scm, "gc", @args)
                unless $scm eq "darcs";
        }
Ian Lynagh's avatar
Ian Lynagh committed
455 456 457
        elsif ($command =~ /^tag$/) {
            scm ($localpath, $scm, "tag", @args);
        }
Ian Lynagh's avatar
Ian Lynagh committed
458 459 460
        else {
            die "Unknown command: $command";
        }
461
    }
462 463

    unlink "resume";
464 465
}

466
sub help
467
{
468 469
        my $exit = shift;

470 471
        # Get the built in help
        my $help = <<END;
472 473
Usage:

474
./sync-all [-q] [-s] [--ignore-failure] [-r repo] [--checked-out] [--bare]
dterei's avatar
dterei committed
475 476
           [--nofib] [--extra] [--testsuite] [--no-dph] [--resume]
           cmd [git flags]
477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502

Applies the command "cmd" to each repository in the tree.

A full repository tree is obtained by first cloning the ghc
repository, then getting the subrepositories with "sync-all get":

  \$ git clone http://darcs.haskell.org/ghc.git
  \$ cd ghc
  \$ ./sync-all get

After this, "./sync-all pull" will pull from the original repository
tree.

A remote pointing to another local repository tree can be added like
this:

  \$ ./sync-all -r /path/to/ghc remote add otherlocal

and then we can pull from this other tree with

  \$ ./sync-all pull otherlocal

-------------- Commands -----------------
get

    Clones all sub-repositories from the same place that the ghc
dterei's avatar
dterei committed
503
    repository was cloned from. See "which repos to use" below
504 505 506
    for details of how the subrepositories are laid out.

    There are various --<package-tag> options that can be given
dterei's avatar
dterei committed
507 508
    before "get" that enable extra repositories. The full list is
    given at the end of this help. For example:
509 510 511 512 513 514 515 516 517 518 519

    ./sync-all --testsuite get

    would get the testsuite repository in addition to the usual set of
    subrepositories.

remote add <remote-name>
remote rm <remote-name>
remote set-url [--push] <remote-name>

    Runs a "git remote" command on each subrepository, adjusting the
dterei's avatar
dterei committed
520
    repository location in each case appropriately. For example, to
521 522 523 524 525
    add a new remote pointing to the upstream repositories:

    ./sync-all -r http://darcs.haskell.org/ remote add upstream

    The -r flag points to the root of the repository tree (see "which
dterei's avatar
dterei committed
526
    repos to use" below). For a repository on the local filesystem it
Simon Marlow's avatar
typo  
Simon Marlow committed
527
    would point to the ghc repository, and for a remote repository it
528 529 530 531 532
    points to the directory containing "ghc.git".

These commands just run the equivalent git command on each repository, passing
any extra arguments to git:

533
  branch
534 535 536 537
  checkout
  clean
  commit
  config
Ian Lynagh's avatar
Ian Lynagh committed
538
  diff
539
  fetch
540
  format-patch
541
  gc
542 543 544
  grep
  log
  new
tibbe's avatar
tibbe committed
545
  new-workdir
546 547
  pull
  push
548
  repack
549 550 551
  reset
  send
  status
Ian Lynagh's avatar
Ian Lynagh committed
552
  tag
553 554

-------------- Flags -------------------
dterei's avatar
dterei committed
555 556
These flags are given *before* the command and modify the way sync-all behaves.
Flags given *after* the command are passed to git.
557

daniel.is.fischer's avatar
daniel.is.fischer committed
558
  -q says to be quiet, and -s to be silent.
559

dterei's avatar
dterei committed
560 561 562
  --resume will restart a command that failed, from the repo at which it
  failed. This means you don't need to wait while, e.g., "pull" goes through
  all the repos it's just pulled, and tries to pull them again.
563 564 565 566 567

  --ignore-failure says to ignore errors and move on to the next repository

  -r repo says to use repo as the location of package repositories

dterei's avatar
dterei committed
568 569 570 571 572
  --checked-out says that the remote repo is in checked-out layout, as opposed
  to the layout used for the main repo. By default a repo on the local
  filesystem is assumed to be checked-out, and repos accessed via HTTP or SSH
  are assumed to be in the main repo layout; use --checked-out to override the
  latter.
573

dterei's avatar
dterei committed
574 575 576
  --bare says that the local repo is in bare layout, same as the main repo. It
  also means that these repos are bare. You only have to use this flag if you
  don't have a bare ghc.git in the current directory and would like to 'get'
577 578 579
  all of the repos bare. Requires packages.conf to be present in the current
  directory (a renamed packages file from the main ghc repo).

daniel.is.fischer's avatar
daniel.is.fischer committed
580
  Note: --checked-out and --bare flags are NOT the opposite of each other.
581 582 583
        --checked-out: describes the layout of the remote repository tree.
        --bare:        describes the layout of the local repository tree.

dterei's avatar
dterei committed
584 585 586 587 588 589 590 591 592 593 594 595 596
  --nofib also clones the nofib benchmark suite

  --testsuite also clones the ghc testsuite 

  --extra also clone some extra library packages

  --no-dph avoids cloning the dph pacakges


------------ Checking out a branch -------------
To check out a branch you can run the following command:

  \$ ./sync-all checkout ghc-7.4
597 598 599


------------ Which repos to use -------------
dterei's avatar
dterei committed
600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
sync-all uses the following algorithm to decide which remote repos to use

It always computes the remote repos from a single base, <repo_base> How is
<repo_base> set? If you say "-r repo", then that's <repo_base> otherwise
<repo_base> is set by asking git where the ghc repo came from, and removing the
last component (e.g. /ghc.git/ or /ghc/).

Then sync-all iterates over the package found in the file ./packages; see that
file for a description of the contents.

If <repo_base> looks like a local filesystem path, or if you give the
--checked-out flag, sync-all works on repos of form:

  <repo_base>/<local-path>

otherwise sync-all works on repos of form:

  <repo_base>/<remote-path>

This logic lets you say
  both    sync-all -r http://darcs.haskell.org/ghc-6.12 remote add ghc-6.12
  and     sync-all -r ../working remote add working
The latter is called a "checked-out tree".

sync-all *ignores* the defaultrepo of all repos other than the root one. So the
remote repos must be laid out in one of the two formats given by <local-path>
and <remote-path> in the file 'packages'.
627

628 629 630 631 632
Available package-tags are:
END

        # Collect all the tags in the packages file
        my %available_tags;
633 634 635
        open IN, "< packages.conf"
            or open IN, "< packages" # clashes with packages directory when using --bare
            or die "Can't open packages file (or packages.conf)";
636 637 638 639 640 641 642 643 644 645 646 647
        while (<IN>) {
            chomp;
            if (/^([^# ]+) +(?:([^ ]+) +)?([^ ]+) +([^ ]+)/) {
                if (defined($2) && $2 ne "-") {
                    $available_tags{$2} = 1;
                }
            }
            elsif (! /^(#.*)?$/) {
                die "Bad line: $_";
            }
        }
        close IN;
daniel.is.fischer's avatar
daniel.is.fischer committed
648

649 650
        # Show those tags and the help text
        my @available_tags = keys %available_tags;
651 652
        print "$help@available_tags\n\n";
        exit $exit;
653 654
}

655 656
sub main {

Simon Marlow's avatar
Simon Marlow committed
657 658
    $tags{"-"} = 1;

659 660 661 662 663 664 665 666 667 668
    while ($#_ ne -1) {
        my $arg = shift;
        # We handle -q here as well as lower down as we need to skip over it
        # if it comes before the source-control command
        if ($arg eq "-q") {
            $verbose = 1;
        }
        elsif ($arg eq "-s") {
            $verbose = 0;
        }
Simon Marlow's avatar
Simon Marlow committed
669 670 671
        elsif ($arg eq "-r") {
            $defaultrepo = shift;
        }
672 673 674
        elsif ($arg eq "--resume") {
            $try_to_resume = 1;
        }
675 676 677
        elsif ($arg eq "--ignore-failure") {
            $ignore_failure = 1;
        }
678 679 680
        elsif ($arg eq "--complete" || $arg eq "--partial") {
            $get_mode = $arg;
        }
681
        # Use --checked-out if the _remote_ repos are a checked-out tree,
Simon Marlow's avatar
Simon Marlow committed
682 683 684 685
        # rather than the master trees.
        elsif ($arg eq "--checked-out") {
            $checked_out_flag = 1;
        }
686 687 688 689 690
        # Use --bare if the _local_ repos are bare repos,
        # rather than a checked-out tree.
        elsif ($arg eq "--bare") {
            $bare_flag = $arg;
        }
691 692 693
        elsif ($arg eq "--help") {
            help(0);
        }
694 695
        # --<tag> says we grab the libs tagged 'tag' with
        # 'get'. It has no effect on the other commands.
696 697 698 699 700
        elsif ($arg =~ m/^--no-(.*)$/) {
            $tags{$1} = 0;
        }
        elsif ($arg =~ m/^--(.*)$/) {
            $tags{$1} = 1;
701 702 703 704 705 706 707 708 709 710
        }
        else {
            unshift @_, $arg;
            if (grep /^-q$/, @_) {
                $verbose = 1;
            }
            last;
        }
    }

711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
    # check for ghc repositories in cwd
    my $checked_out_found = 1 if (-d ".git" && -d "compiler");
    my $bare_found = 1 if (-d "ghc.git");

    if ($bare_flag && ! $bare_found && ! $defaultrepo) {
        die "error: bare repository ghc.git not found.\n"
          . "       Either clone a bare ghc repo first or specify the repo location. E.g.:\n"
          . "       ./sync-all --bare [--testsuite --nofib --extra] -r http://darcs.haskell.org/ get\n"
    }
    elsif ($bare_found) {
        $bare_flag = "--bare";
    }
    elsif (! $bare_flag && ! $checked_out_found) {
        die "error: sync-all must be run from the top level of the ghc tree.";
    }

727
    if ($#_ eq -1) {
728
        help(1);
729 730 731 732 733 734 735
    }
    else {
        # Give the command and rest of the arguments to the main loop
        scmall @_;
    }
}

736 737 738 739 740 741 742 743 744 745 746 747 748 749 750
END {
    my $ec = $?;
    my $pwd = getcwd();

    message "== Checking for old haddock repo";
    if (-d "utils/haddock/.git") {
        chdir("utils/haddock");
        if ((system "git log -1 87e2ca11c3d1b1bc49900fba0b5c5c6f85650718 > /dev/null 2> /dev/null") == 0) {
            print <<EOF;
============================
ATTENTION!

You have an old haddock repository in your GHC tree!

Please remove it (e.g. "rm -r utils/haddock"), and then run
daniel.is.fischer's avatar
daniel.is.fischer committed
751
"./sync-all get" to get the new repository.
752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
============================
EOF
        }
        chdir($pwd);
    }

    message "== Checking for old binary repo";
    if (-d "libraries/binary/.git") {
        chdir("libraries/binary");
        if ((system "git log -1 749ac0efbde3b14901417364a872796598747aaf > /dev/null 2> /dev/null") == 0) {
            print <<EOF;
============================
ATTENTION!

You have an old binary repository in your GHC tree!

Please remove it (e.g. "rm -r libraries/binary"), and then run
daniel.is.fischer's avatar
daniel.is.fischer committed
769
"./sync-all get" to get the new repository.
770 771 772 773 774 775
============================
EOF
        }
        chdir($pwd);
    }

776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
    message "== Checking for old mtl repo";
    if (-d "libraries/mtl/.git") {
        chdir("libraries/mtl");
        if ((system "git log -1 c67d8f7247c612dc35242bc67e616f7ea35eadb9 > /dev/null 2> /dev/null") == 0) {
            print <<EOF;
============================
ATTENTION!

You have an old mtl repository in your GHC tree!

Please remove it (e.g. "rm -r libraries/mtl"), and then run
"./sync-all get" to get the new repository.
============================
EOF
        }
        chdir($pwd);
    }

    message "== Checking for old Cabal repo";
    if (-d "libraries/Cabal/.git") {
        chdir("libraries/Cabal");
        if ((system "git log -1 c8ebd66a32865f72ae03ee0663c62df3d77f08fe > /dev/null 2> /dev/null") == 0) {
            print <<EOF;
============================
ATTENTION!

You have an old Cabal repository in your GHC tree!

Please remove it (e.g. "rm -r libraries/Cabal"), and then run
"./sync-all get" to get the new repository.
============================
EOF
        }
        chdir($pwd);
    }

812 813 814
    $? = $ec;
}

815 816
main(@ARGV);