build/unix/build-clang/build-clang.py@b8a032363ba2 (annotated)
build/unix/build-clang/build-clang.py
Thu, 22 Jan 2015 13:21:57 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Thu, 22 Jan 2015 13:21:57 +0100
- branch
- TOR_BUG_9701
- changeset 15
- b8a032363ba2
- permissions
- -rwxr-xr-x
Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6
| michael@0 | 1 | #!/usr/bin/python |
| michael@0 | 2 | # This Source Code Form is subject to the terms of the Mozilla Public |
| michael@0 | 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this |
| michael@0 | 4 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| michael@0 | 5 | |
| michael@0 | 6 | import os |
| michael@0 | 7 | import os.path |
| michael@0 | 8 | import shutil |
| michael@0 | 9 | import subprocess |
| michael@0 | 10 | import platform |
| michael@0 | 11 | import sys |
| michael@0 | 12 | import json |
| michael@0 | 13 | import collections |
| michael@0 | 14 | import argparse |
| michael@0 | 15 | |
| michael@0 | 16 | |
| michael@0 | 17 | def check_run(args): |
| michael@0 | 18 | r = subprocess.call(args) |
| michael@0 | 19 | assert r == 0 |
| michael@0 | 20 | |
| michael@0 | 21 | |
| michael@0 | 22 | def run_in(path, args): |
| michael@0 | 23 | d = os.getcwd() |
| michael@0 | 24 | os.chdir(path) |
| michael@0 | 25 | check_run(args) |
| michael@0 | 26 | os.chdir(d) |
| michael@0 | 27 | |
| michael@0 | 28 | |
| michael@0 | 29 | def patch(patch, srcdir): |
| michael@0 | 30 | patch = os.path.realpath(patch) |
| michael@0 | 31 | check_run(['patch', '-d', srcdir, '-p1', '-i', patch, '--fuzz=0', |
| michael@0 | 32 | '-s']) |
| michael@0 | 33 | |
| michael@0 | 34 | |
| michael@0 | 35 | def build_package(package_source_dir, package_build_dir, configure_args, |
| michael@0 | 36 | make_args): |
| michael@0 | 37 | if not os.path.exists(package_build_dir): |
| michael@0 | 38 | os.mkdir(package_build_dir) |
| michael@0 | 39 | run_in(package_build_dir, |
| michael@0 | 40 | ["%s/configure" % package_source_dir] + configure_args) |
| michael@0 | 41 | run_in(package_build_dir, ["make", "-j4"] + make_args) |
| michael@0 | 42 | run_in(package_build_dir, ["make", "install"]) |
| michael@0 | 43 | |
| michael@0 | 44 | |
| michael@0 | 45 | def with_env(env, f): |
| michael@0 | 46 | old_env = os.environ.copy() |
| michael@0 | 47 | os.environ.update(env) |
| michael@0 | 48 | f() |
| michael@0 | 49 | os.environ.clear() |
| michael@0 | 50 | os.environ.update(old_env) |
| michael@0 | 51 | |
| michael@0 | 52 | |
| michael@0 | 53 | def build_tar_package(tar, name, base, directory): |
| michael@0 | 54 | name = os.path.realpath(name) |
| michael@0 | 55 | run_in(base, [tar, "-cjf", name, directory]) |
| michael@0 | 56 | |
| michael@0 | 57 | |
| michael@0 | 58 | def svn_co(url, directory, revision): |
| michael@0 | 59 | check_run(["svn", "co", "-r", revision, url, directory]) |
| michael@0 | 60 | |
| michael@0 | 61 | |
| michael@0 | 62 | def build_one_stage(env, stage_dir, llvm_source_dir, gcc_toolchain_dir): |
| michael@0 | 63 | def f(): |
| michael@0 | 64 | build_one_stage_aux(stage_dir, llvm_source_dir, gcc_toolchain_dir) |
| michael@0 | 65 | with_env(env, f) |
| michael@0 | 66 | |
| michael@0 | 67 | |
| michael@0 | 68 | def build_tooltool_manifest(llvm_revision): |
| michael@0 | 69 | basedir = os.path.split(os.path.realpath(sys.argv[0]))[0] |
| michael@0 | 70 | tooltool = basedir + '/tooltool.py' |
| michael@0 | 71 | setup = basedir + '/setup.sh' |
| michael@0 | 72 | manifest = 'clang.manifest' |
| michael@0 | 73 | check_run(['python', tooltool, '-m', manifest, 'add', |
| michael@0 | 74 | setup, 'clang.tar.bz2']) |
| michael@0 | 75 | data = json.load(file(manifest), object_pairs_hook=collections.OrderedDict) |
| michael@0 | 76 | data = [{'clang_version': 'r%s' % llvm_revision}] + data |
| michael@0 | 77 | out = file(manifest, 'w') |
| michael@0 | 78 | json.dump(data, out, indent=0) |
| michael@0 | 79 | out.write('\n') |
| michael@0 | 80 | |
| michael@0 | 81 | assert data[2]['filename'] == 'clang.tar.bz2' |
| michael@0 | 82 | os.rename('clang.tar.bz2', data[2]['digest']) |
| michael@0 | 83 | |
| michael@0 | 84 | |
| michael@0 | 85 | def get_platform(): |
| michael@0 | 86 | p = platform.system() |
| michael@0 | 87 | if p == "Darwin": |
| michael@0 | 88 | return "macosx64" |
| michael@0 | 89 | elif p == "Linux": |
| michael@0 | 90 | if platform.processor() == "x86_64": |
| michael@0 | 91 | return "linux64" |
| michael@0 | 92 | else: |
| michael@0 | 93 | return "linux32" |
| michael@0 | 94 | else: |
| michael@0 | 95 | raise NotImplementedError("Not supported platform") |
| michael@0 | 96 | |
| michael@0 | 97 | |
| michael@0 | 98 | def is_darwin(): |
| michael@0 | 99 | return platform.system() == "Darwin" |
| michael@0 | 100 | |
| michael@0 | 101 | |
| michael@0 | 102 | def build_one_stage_aux(stage_dir, llvm_source_dir, gcc_toolchain_dir): |
| michael@0 | 103 | os.mkdir(stage_dir) |
| michael@0 | 104 | |
| michael@0 | 105 | build_dir = stage_dir + "/build" |
| michael@0 | 106 | inst_dir = stage_dir + "/clang" |
| michael@0 | 107 | |
| michael@0 | 108 | targets = ["x86", "x86_64"] |
| michael@0 | 109 | # The Darwin equivalents of binutils appear to have intermittent problems |
| michael@0 | 110 | # with objects in compiler-rt that are compiled for arm. Since the arm |
| michael@0 | 111 | # support is only necessary for iOS (which we don't support), only enable |
| michael@0 | 112 | # arm support on Linux. |
| michael@0 | 113 | if not is_darwin(): |
| michael@0 | 114 | targets.append("arm") |
| michael@0 | 115 | |
| michael@0 | 116 | configure_opts = ["--enable-optimized", |
| michael@0 | 117 | "--enable-targets=" + ",".join(targets), |
| michael@0 | 118 | "--disable-assertions", |
| michael@0 | 119 | "--prefix=%s" % inst_dir, |
| michael@0 | 120 | "--with-gcc-toolchain=%s" % gcc_toolchain_dir, |
| michael@0 | 121 | "--disable-compiler-version-checks"] |
| michael@0 | 122 | build_package(llvm_source_dir, build_dir, configure_opts, []) |
| michael@0 | 123 | |
| michael@0 | 124 | if __name__ == "__main__": |
| michael@0 | 125 | # The directories end up in the debug info, so the easy way of getting |
| michael@0 | 126 | # a reproducible build is to run it in a know absolute directory. |
| michael@0 | 127 | # We use a directory in /builds/slave because the mozilla infrastructure |
| michael@0 | 128 | # cleans it up automatically. |
| michael@0 | 129 | base_dir = "/builds/slave/moz-toolchain" |
| michael@0 | 130 | |
| michael@0 | 131 | source_dir = base_dir + "/src" |
| michael@0 | 132 | build_dir = base_dir + "/build" |
| michael@0 | 133 | |
| michael@0 | 134 | llvm_source_dir = source_dir + "/llvm" |
| michael@0 | 135 | clang_source_dir = source_dir + "/clang" |
| michael@0 | 136 | compiler_rt_source_dir = source_dir + "/compiler-rt" |
| michael@0 | 137 | |
| michael@0 | 138 | gcc_dir = "/tools/gcc-4.7.3-0moz1" |
| michael@0 | 139 | |
| michael@0 | 140 | if is_darwin(): |
| michael@0 | 141 | os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.7' |
| michael@0 | 142 | |
| michael@0 | 143 | parser = argparse.ArgumentParser() |
| michael@0 | 144 | parser.add_argument('-c', '--config', required=True, |
| michael@0 | 145 | type=argparse.FileType('r'), |
| michael@0 | 146 | help="Clang configuration file") |
| michael@0 | 147 | |
| michael@0 | 148 | args = parser.parse_args() |
| michael@0 | 149 | config = json.load(args.config) |
| michael@0 | 150 | llvm_revision = config["llvm_revision"] |
| michael@0 | 151 | llvm_repo = config["llvm_repo"] |
| michael@0 | 152 | clang_repo = config["clang_repo"] |
| michael@0 | 153 | compiler_repo = config["compiler_repo"] |
| michael@0 | 154 | |
| michael@0 | 155 | if not os.path.exists(source_dir): |
| michael@0 | 156 | os.makedirs(source_dir) |
| michael@0 | 157 | svn_co(llvm_repo, llvm_source_dir, llvm_revision) |
| michael@0 | 158 | svn_co(clang_repo, clang_source_dir, llvm_revision) |
| michael@0 | 159 | svn_co(compiler_repo, compiler_rt_source_dir, llvm_revision) |
| michael@0 | 160 | os.symlink("../../clang", llvm_source_dir + "/tools/clang") |
| michael@0 | 161 | os.symlink("../../compiler-rt", |
| michael@0 | 162 | llvm_source_dir + "/projects/compiler-rt") |
| michael@0 | 163 | for p in config.get("patches", {}).get(get_platform(), []): |
| michael@0 | 164 | patch(p, source_dir) |
| michael@0 | 165 | |
| michael@0 | 166 | if os.path.exists(build_dir): |
| michael@0 | 167 | shutil.rmtree(build_dir) |
| michael@0 | 168 | os.makedirs(build_dir) |
| michael@0 | 169 | |
| michael@0 | 170 | stage1_dir = build_dir + '/stage1' |
| michael@0 | 171 | stage1_inst_dir = stage1_dir + '/clang' |
| michael@0 | 172 | |
| michael@0 | 173 | if is_darwin(): |
| michael@0 | 174 | extra_cflags = "" |
| michael@0 | 175 | extra_cxxflags = "" |
| michael@0 | 176 | cc = "/usr/bin/clang" |
| michael@0 | 177 | cxx = "/usr/bin/clang++" |
| michael@0 | 178 | else: |
| michael@0 | 179 | extra_cflags = "-static-libgcc" |
| michael@0 | 180 | extra_cxxflags = "-static-libgcc -static-libstdc++" |
| michael@0 | 181 | cc = gcc_dir + "/bin/gcc" |
| michael@0 | 182 | cxx = gcc_dir + "/bin/g++" |
| michael@0 | 183 | |
| michael@0 | 184 | if os.environ.has_key('LD_LIBRARY_PATH'): |
| michael@0 | 185 | os.environ['LD_LIBRARY_PATH'] = '%s/lib64/:%s' % (gcc_dir, os.environ['LD_LIBRARY_PATH']); |
| michael@0 | 186 | else: |
| michael@0 | 187 | os.environ['LD_LIBRARY_PATH'] = '%s/lib64/' % gcc_dir |
| michael@0 | 188 | |
| michael@0 | 189 | build_one_stage({"CC": cc, "CXX": cxx}, stage1_dir, llvm_source_dir, gcc_dir) |
| michael@0 | 190 | |
| michael@0 | 191 | stage2_dir = build_dir + '/stage2' |
| michael@0 | 192 | build_one_stage( |
| michael@0 | 193 | {"CC": stage1_inst_dir + "/bin/clang %s" % extra_cflags, |
| michael@0 | 194 | "CXX": stage1_inst_dir + "/bin/clang++ %s" % extra_cxxflags}, |
| michael@0 | 195 | stage2_dir, llvm_source_dir, gcc_dir) |
| michael@0 | 196 | |
| michael@0 | 197 | build_tar_package("tar", "clang.tar.bz2", stage2_dir, "clang") |
| michael@0 | 198 | build_tooltool_manifest(llvm_revision) |
