diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f4600a..65a669c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ jobs: test: runs-on: macOS-10.15 steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3 - name: Run tests env: DEVELOPER_DIR: /Applications/Xcode_12.3.app diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme index f2745c4..165d010 100644 --- a/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme +++ b/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme @@ -90,6 +90,34 @@ ReferencedContainer = "container:"> + + + + + + + + /dev/null || true - @zip --junk-paths xcodes.zip "$(BUILDDIR)/release/xcodes" + @zip --junk-paths xcodes.zip "$(RELEASEBUILDDIR)" @open -R xcodes.zip # E.g. -# make notarize USERNAME="test@example.com" PASSWORD="@keychain:altool" ASC_PROVIDER=MyOrg +# make notarize TEAMID="ABCD123" .PHONY: notarize notarize: zip - ./notarize.sh "$(USERNAME)" "$(PASSWORD)" "$(ASC_PROVIDER)" xcodes.zip + ./notarize.sh xcodes.zip "$(TEAMID)" # E.g. # make bottle VERSION=0.4.0 @@ -54,7 +54,7 @@ bottle: sign @rm -r xcodes 2> /dev/null || true @rm *.tar.gz 2> /dev/null || true @mkdir -p xcodes/$(VERSION)/bin - @cp "$(BUILDDIR)/release/xcodes" xcodes/$(VERSION)/bin + @cp "$(RELEASEBUILDDIR)" xcodes/$(VERSION)/bin @tar -zcvf xcodes-$(VERSION).mojave.bottle.tar.gz -C "$(REPODIR)" xcodes shasum -a 256 xcodes-$(VERSION).mojave.bottle.tar.gz | cut -f1 -d' ' @open -R xcodes-$(VERSION).mojave.bottle.tar.gz @@ -62,7 +62,7 @@ bottle: sign .PHONY: install install: xcodes @install -d "$(bindir)" - @install "$(BUILDDIR)/release/xcodes" "$(bindir)" + @install "$(RELEASEBUILDDIR)" "$(bindir)" .PHONY: uninstall uninstall: @@ -70,7 +70,7 @@ uninstall: .PHONY: distclean distclean: - @rm -f $(BUILDDIR)/release + @rm -f $(RELEASEBUILDDIR) .PHONY: clean clean: distclean diff --git a/Package.swift b/Package.swift index 29a1343..4a466f7 100644 --- a/Package.swift +++ b/Package.swift @@ -45,6 +45,7 @@ let package = Package( "PromiseKit", "PMKFoundation", "SwiftSoup", + "Unxip", "Version", .product(name: "XCModel", package: "XcodeReleases"), "Rainbow", @@ -58,6 +59,7 @@ let package = Package( resources: [ .copy("Fixtures"), ]), + .target(name: "Unxip"), .target( name: "AppleAPI", dependencies: [ diff --git a/README.md b/README.md index 0b0ce97..c8b13b5 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ xcodes install 11.2 GM seed xcodes install 9.0 --path ~/Archive/Xcode_9.xip xcodes install --latest-prerelease xcodes install --latest --directory "/Volumes/Bag Of Holding/" +xcodes install --latest --experimental-unxip ``` You'll then be prompted to enter your Apple ID username and password. You can also provide these with the `XCODES_USERNAME` and `XCODES_PASSWORD` environment variables. @@ -106,6 +107,14 @@ Xcode will be installed to /Applications by default, but you can provide the pat - `version`: Print the version number of xcodes itself - `signout`: Clears the stored username and password +### Experimental Unxip - for faster unxipping + +Thanks to the amazing work by [saagarjhi](https://github.com/saagarjha/unxip) - Xcodes now includes the ability to unxip up to 70% faster on some systems. + +``` +xcodes install --latest --experimental-unxip +``` + ### Shell Completion Scripts xcodes can generate completion scripts which allow you to press the tab key on your keyboard to autocomplete commands and arguments when typing an xcodes command. The steps to install a completion script depend on the shell that you use. More information about installation instructions for different shells and the underlying implementation is available in the [swift-argument-parser repo](https://github.com/apple/swift-argument-parser/blob/main/Sources/ArgumentParser/Documentation.docc/Articles/InstallingCompletionScripts.md). @@ -123,7 +132,7 @@ xcodes --generate-completion-script > ~/.oh-my-zsh/completions/_xcodes ## Development -You'll need Xcode 12 in order to build and run xcodes. +You'll need Xcode 13 in order to build and run xcodes.
Using Xcode @@ -166,9 +175,7 @@ make bottle VERSION="$VERSION" # Notarize the release build # This can take a while make notarize \ - USERNAME="user@example.com" \ - PASSWORD="@keychain:ALTool Notarization" \ - ASC_PROVIDER="YourAppStoreConnectTeamName" + TEAMID="ABC123" # Push the new version bump commit and tag git push --follow-tags diff --git a/Sources/Unxip/Unxip.swift b/Sources/Unxip/Unxip.swift new file mode 100644 index 0000000..a078bd6 --- /dev/null +++ b/Sources/Unxip/Unxip.swift @@ -0,0 +1,513 @@ +import Compression +import Foundation + +// From: https://github.com/saagarjha/unxip +// License: GNU Lesser General Public License v3.0 + +extension RandomAccessCollection { + subscript(fromOffset fromOffset: Int = 0, toOffset toOffset: Int? = nil) -> SubSequence { + let toOffset = toOffset ?? count + return self[index(startIndex, offsetBy: fromOffset).. SubSequence { + let base = index(startIndex, offsetBy: fromOffset) + return self[base.. { + let batchSize: Int + var operations = [@Sendable () async throws -> TaskResult]() + + var results: AsyncStream { + AsyncStream(bufferingPolicy: .bufferingOldest(batchSize)) { continuation in + Task { + try await withThrowingTaskGroup(of: (Int, TaskResult).self) { group in + var queueIndex = 0 + var dequeIndex = 0 + var pending = [Int: TaskResult]() + while dequeIndex < operations.count { + if queueIndex - dequeIndex < batchSize, + queueIndex < operations.count + { + let _queueIndex = queueIndex + group.addTask { + let queueIndex = _queueIndex + return await (queueIndex, try operations[queueIndex]()) + } + queueIndex += 1 + } else { + let (index, result) = try await group.next()! + pending[index] = result + if index == dequeIndex { + while let result = pending[dequeIndex] { + await continuation.yieldWithBackoff(result) + pending.removeValue(forKey: dequeIndex) + dequeIndex += 1 + } + } + } + } + continuation.finish() + } + } + } + } + + init(batchSize: Int = ProcessInfo.processInfo.activeProcessorCount) { + self.batchSize = batchSize + } + + mutating func addTask(operation: @escaping @Sendable () async throws -> TaskResult) { + operations.append(operation) + } + + mutating func addRunningTask(operation: @escaping @Sendable () async -> TaskResult) -> Task { + let task = Task { + await operation() + } + operations.append({ + await task.value + }) + return task + } +} + +final class Chunk: Sendable { + let buffer: UnsafeBufferPointer + let owned: Bool + + init(buffer: UnsafeBufferPointer, owned: Bool) { + self.buffer = buffer + self.owned = owned + } + + deinit { + if owned { + buffer.deallocate() + } + } +} + +@available(macOS 11, *) +struct File { + let dev: Int + let ino: Int + let mode: Int + let name: String + var data = [UnsafeBufferPointer]() + // For keeping the data alive + var chunks = [Chunk]() + + struct Identifier: Hashable { + let dev: Int + let ino: Int + } + + var identifier: Identifier { + Identifier(dev: dev, ino: ino) + } + + func writeCompressedIfPossible(usingDescriptor descriptor: CInt) async -> Bool { + let blockSize = 64 << 10 // LZFSE with 64K block size + var _data = [UInt8]() + _data.reserveCapacity(self.data.map(\.count).reduce(0, +)) + let data = self.data.reduce(into: _data, +=) + var compressionStream = ConcurrentStream<[UInt8]?>() + var position = data.startIndex + + while position < data.endIndex { + let _position = position + compressionStream.addTask { + try Task.checkCancellation() + let position = _position + let data = [UInt8](unsafeUninitializedCapacity: blockSize + blockSize / 16) { buffer, count in + data[position...size + let size = tableSize + chunks.map(\.count).reduce(0, +) + guard size < data.count else { + return false + } + + let buffer = [UInt8](unsafeUninitializedCapacity: size) { buffer, count in + var position = tableSize + + func writePosition(toTableIndex index: Int) { + precondition(position < UInt32.max) + for i in 0...size { + buffer[index * MemoryLayout.size + i] = UInt8(position >> (i * 8) & 0xff) + } + } + + writePosition(toTableIndex: 0) + for (index, chunk) in zip(1..., chunks) { + _ = UnsafeMutableBufferPointer(rebasing: buffer.suffix(from: position)).initialize(from: chunk) + position += chunk.count + writePosition(toTableIndex: index) + } + count = size + } + + let attribute = + "cmpf".utf8.reversed() // magic + + [0x0c, 0x00, 0x00, 0x00] // LZFSE, 64K chunks + + ([ + (data.count >> 0) & 0xff, + (data.count >> 8) & 0xff, + (data.count >> 16) & 0xff, + (data.count >> 24) & 0xff, + (data.count >> 32) & 0xff, + (data.count >> 40) & 0xff, + (data.count >> 48) & 0xff, + (data.count >> 56) & 0xff, + ].map(UInt8.init) as [UInt8]) + + guard fsetxattr(descriptor, "com.apple.decmpfs", attribute, attribute.count, 0, XATTR_SHOWCOMPRESSION) == 0 else { + return false + } + + let resourceForkDescriptor = open(name + _PATH_RSRCFORKSPEC, O_WRONLY | O_CREAT, 0o666) + guard resourceForkDescriptor >= 0 else { + return false + } + defer { + close(resourceForkDescriptor) + } + + var written: Int + repeat { + // TODO: handle partial writes smarter + written = pwrite(resourceForkDescriptor, buffer, buffer.count, 0) + guard written >= 0 else { + return false + } + } while written != buffer.count + + guard fchflags(descriptor, UInt32(UF_COMPRESSED)) == 0 else { + return false + } + + return true + } +} + +public struct UnxipOptions { + var input: URL + var output: URL? + + public init(input: URL, output: URL) { + self.input = input + self.output = output + } +} + +@available(macOS 11, *) +public struct Unxip { + let options: UnxipOptions + + public init(options: UnxipOptions) { + self.options = options + } + + func read(_ type: Integer.Type, from buffer: inout Buffer) -> Integer where Buffer.Element == UInt8, Buffer.SubSequence == Buffer { + defer { + buffer = buffer[fromOffset: MemoryLayout.size] + } + var result: Integer = 0 + var iterator = buffer.makeIterator() + for _ in 0...size { + result <<= 8 + result |= Integer(iterator.next()!) + } + return result + } + + func chunks(from content: UnsafeBufferPointer) -> ConcurrentStream { + var remaining = content[fromOffset: 4] + let chunkSize = read(UInt64.self, from: &remaining) + var decompressedSize: UInt64 = 0 + + var chunkStream = ConcurrentStream() + + repeat { + decompressedSize = read(UInt64.self, from: &remaining) + let compressedSize = read(UInt64.self, from: &remaining) + let _remaining = remaining + let _decompressedSize = decompressedSize + + chunkStream.addTask { + let remaining = _remaining + let decompressedSize = _decompressedSize + + if compressedSize == chunkSize { + return Chunk(buffer: UnsafeBufferPointer(rebasing: remaining[fromOffset: 0, size: Int(compressedSize)]), owned: false) + } else { + let magic = [0xfd] + "7zX".utf8 + precondition(remaining.prefix(magic.count).elementsEqual(magic)) + let buffer = UnsafeMutableBufferPointer.allocate(capacity: Int(decompressedSize)) + precondition(compression_decode_buffer(buffer.baseAddress!, buffer.count, UnsafeBufferPointer(rebasing: remaining).baseAddress!, Int(compressedSize), nil, COMPRESSION_LZMA) == decompressedSize) + return Chunk(buffer: UnsafeBufferPointer(buffer), owned: true) + } + } + remaining = remaining[fromOffset: Int(compressedSize)] + } while decompressedSize == chunkSize + + return chunkStream + } + + func files(in chunkStream: ChunkStream) -> AsyncStream where ChunkStream.Element == Chunk { + AsyncStream(bufferingPolicy: .bufferingOldest(ProcessInfo.processInfo.activeProcessorCount)) { continuation in + Task { + var iterator = chunkStream.makeAsyncIterator() + var chunk = try! await iterator.next()! + var position = 0 + + func read(size: Int) async -> [UInt8] { + var result = [UInt8]() + while result.count < size { + if position >= chunk.buffer.endIndex { + chunk = try! await iterator.next()! + position = 0 + } + result.append(chunk.buffer[chunk.buffer.startIndex + position]) + position += 1 + } + return result + } + + func readOctal(from bytes: [UInt8]) -> Int { + Int(String(data: Data(bytes), encoding: .utf8)!, radix: 8)! + } + + while true { + let magic = await read(size: 6) + // Yes, cpio.h really defines this global macro + precondition(magic.elementsEqual(MAGIC.utf8)) + let dev = readOctal(from: await read(size: 6)) + let ino = readOctal(from: await read(size: 6)) + let mode = readOctal(from: await read(size: 6)) + let _ = await read(size: 6) // uid + let _ = await read(size: 6) // gid + let _ = await read(size: 6) // nlink + let _ = await read(size: 6) // rdev + let _ = await read(size: 11) // mtime + let namesize = readOctal(from: await read(size: 6)) + var filesize = readOctal(from: await read(size: 11)) + let name = String(cString: await read(size: namesize)) + var file = File(dev: dev, ino: ino, mode: mode, name: name) + + while filesize > 0 { + if position >= chunk.buffer.endIndex { + chunk = try! await iterator.next()! + position = 0 + } + let size = min(filesize, chunk.buffer.endIndex - position) + file.chunks.append(chunk) + file.data.append(UnsafeBufferPointer(rebasing: chunk.buffer[fromOffset: position, size: size])) + filesize -= size + position += size + } + + guard file.name != "TRAILER!!!" else { + continuation.finish() + return + } + + await continuation.yieldWithBackoff(file) + } + } + } + } + + func parseContent(_ content: UnsafeBufferPointer) async { + var taskStream = ConcurrentStream(batchSize: 64) // Worst case, should allow for files up to 64 * 16MB = 1GB + var hardlinks = [File.Identifier: (String, Task)]() + var directories = [Substring: Task]() + for await file in files(in: chunks(from: content).results) { + @Sendable + func warn(_ result: CInt, _ operation: String) { + if result != 0 { + perror("\(operation) \(file.name) failed") + } + } + + // The assumption is that all directories are provided without trailing slashes + func parentDirectory(of path: S) -> S.SubSequence { + return path[.. Task? { + directories[parentDirectory(of: file.name)] ?? directories[String(parentDirectory(of: file.name))[...]] + } + + @Sendable + func setStickyBit(on file: File) { + if file.mode & Int(C_ISVTX) != 0 { + warn(chmod(file.name, mode_t(file.mode)), "Setting sticky bit on") + } + } + + if file.name == "." { + continue + } + + if let (original, task) = hardlinks[file.identifier] { + _ = taskStream.addRunningTask { + await task.value + warn(link(original, file.name), "linking") + } + continue + } + + // The types we care about, anyways + let typeMask = Int(C_ISLNK | C_ISDIR | C_ISREG) + switch CInt(file.mode & typeMask) { + case C_ISLNK: + let task = parentDirectoryTask(for: file) + assert(task != nil, file.name) + _ = taskStream.addRunningTask { + warn(symlink(String(data: Data(file.data.map(Array.init).reduce([], +)), encoding: .utf8), file.name), "symlinking") + setStickyBit(on: file) + } + case C_ISDIR: + let task = parentDirectoryTask(for: file) + assert(task != nil || parentDirectory(of: file.name) == ".", file.name) + directories[file.name[...]] = taskStream.addRunningTask { + await task?.value + warn(mkdir(file.name, mode_t(file.mode & 0o777)), "creating directory at") + setStickyBit(on: file) + } + case C_ISREG: + let task = parentDirectoryTask(for: file) + assert(task != nil, file.name) + hardlinks[file.identifier] = ( + file.name, + taskStream.addRunningTask { + await task?.value + + let fd = open(file.name, O_CREAT | O_WRONLY, mode_t(file.mode & 0o777)) + if fd < 0 { + warn(fd, "creating file at") + return + } + defer { + warn(close(fd), "closing") + setStickyBit(on: file) + } + + if await file.writeCompressedIfPossible(usingDescriptor: fd) { + return + } + + // pwritev requires the vector count to be positive + if file.data.count == 0 { + return + } + + var vector = file.data.map { + iovec(iov_base: UnsafeMutableRawPointer(mutating: $0.baseAddress), iov_len: $0.count) + } + let total = file.data.map(\.count).reduce(0, +) + var written = 0 + + repeat { + // TODO: handle partial writes smarter + written = pwritev(fd, &vector, CInt(vector.count), 0) + if written < 0 { + warn(-1, "writing chunk to") + break + } + } while written != total + } + ) + default: + fatalError("\(file.name) with \(file.mode) is a type that is unhandled") + } + } + + // Run through any stragglers + for await _ in taskStream.results { + } + } + + func locateContent(in file: UnsafeBufferPointer) -> UnsafeBufferPointer { + precondition(file.starts(with: "xar!".utf8)) // magic + var header = file[4...] + let headerSize = read(UInt16.self, from: &header) + precondition(read(UInt16.self, from: &header) == 1) // version + let tocCompressedSize = read(UInt64.self, from: &header) + let tocDecompressedSize = read(UInt64.self, from: &header) + _ = read(UInt32.self, from: &header) // checksum + + let toc = [UInt8](unsafeUninitializedCapacity: Int(tocDecompressedSize)) { buffer, count in + let zlibSkip = 2 // Apple's decoder doesn't want to see CMF/FLG (see RFC 1950) + count = compression_decode_buffer(buffer.baseAddress!, Int(tocDecompressedSize), file.baseAddress! + Int(headerSize) + zlibSkip, Int(tocCompressedSize) - zlibSkip, nil, COMPRESSION_ZLIB) + precondition(count == Int(tocDecompressedSize)) + } + + let document = try! XMLDocument(data: Data(toc)) + let content = try! document.nodes(forXPath: "xar/toc/file").first { + try! $0.nodes(forXPath: "name").first!.stringValue! == "Content" + }! + let contentOffset = Int(try! content.nodes(forXPath: "data/offset").first!.stringValue!)! + let contentSize = Int(try! content.nodes(forXPath: "data/length").first!.stringValue!)! + let contentBase = Int(headerSize) + Int(tocCompressedSize) + contentOffset + + let slice = file[fromOffset: contentBase, size: contentSize] + precondition(slice.starts(with: "pbzx".utf8)) + return UnsafeBufferPointer(rebasing: slice) + } + + public func run() async throws { + let handle = try FileHandle(forReadingFrom: options.input) + try handle.seekToEnd() + let length = Int(try handle.offset()) + let file = UnsafeBufferPointer(start: mmap(nil, length, PROT_READ, MAP_PRIVATE, handle.fileDescriptor, 0).bindMemory(to: UInt8.self, capacity: length), count: length) + precondition(UnsafeMutableRawPointer(mutating: file.baseAddress) != MAP_FAILED) + defer { + munmap(UnsafeMutableRawPointer(mutating: file.baseAddress), length) + } + + if let output = options.output { + guard chdir(output.path) == 0 else { + fputs("Failed to access output directory at \(output.path): \(String(cString: strerror(errno)))", stderr) + exit(EXIT_FAILURE) + } + } + + await parseContent(locateContent(in: file)) + } +} diff --git a/Sources/XcodesKit/Version.swift b/Sources/XcodesKit/Version.swift index 62fcb1b..e722549 100644 --- a/Sources/XcodesKit/Version.swift +++ b/Sources/XcodesKit/Version.swift @@ -1,3 +1,3 @@ import Version -public let version = Version("0.19.0")! +public let version = Version("0.20.0")! diff --git a/Sources/XcodesKit/XcodeInstaller.swift b/Sources/XcodesKit/XcodeInstaller.swift index 6de9403..6b54ec4 100644 --- a/Sources/XcodesKit/XcodeInstaller.swift +++ b/Sources/XcodesKit/XcodeInstaller.swift @@ -5,6 +5,7 @@ import AppleAPI import Version import LegibleError import Rainbow +import Unxip /// Downloads and installs Xcodes public final class XcodeInstaller { @@ -79,7 +80,7 @@ public final class XcodeInstaller { /// A numbered step enum InstallationStep: CustomStringConvertible { case downloading(version: String, progress: String?, willInstall: Bool) - case unarchiving + case unarchiving(experimentalUnxip: Bool) case moving(destination: String) case trashingArchive(archiveName: String) case checkingSecurity @@ -102,8 +103,15 @@ public final class XcodeInstaller { } else { return "Downloading Xcode \(version)" } - case .unarchiving: - return "Unarchiving Xcode (This can take a while)" + case .unarchiving(let experimentalUnxip): + let hint = experimentalUnxip ? + "Using experimental unxip. If you encounter any issues, remove the flag and try again" : + "Using regular unxip. Try passing `--experimental-unxip` for a faster unxip process" + return + """ + Unarchiving Xcode (This can take a while) + \(hint) + """ case .moving(let destination): return "Moving Xcode to \(destination)" case .trashingArchive(let archiveName): @@ -155,9 +163,9 @@ public final class XcodeInstaller { case aria2(Path) } - public func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, shouldExpandXipInplace: Bool) -> Promise { + public func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, experimentalUnxip: Bool = false, shouldExpandXipInplace: Bool) -> Promise { return firstly { () -> Promise in - return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, shouldExpandXipInplace: shouldExpandXipInplace, attemptNumber: 0) + return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: 0, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: shouldExpandXipInplace) } .done { xcode in Current.logging.log("\nXcode \(xcode.version.descriptionWithoutBuildMetadata) has been installed to \(xcode.path.string)".green) @@ -165,12 +173,12 @@ public final class XcodeInstaller { } } - private func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, shouldExpandXipInplace: Bool, attemptNumber: Int) -> Promise { + private func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, attemptNumber: Int, experimentalUnxip: Bool, shouldExpandXipInplace: Bool) -> Promise { return firstly { () -> Promise<(Xcode, URL)> in return self.getXcodeArchive(installationType, dataSource: dataSource, downloader: downloader, destination: destination, willInstall: true) } .then { xcode, url -> Promise in - return self.installArchivedXcode(xcode, at: url, to: destination, shouldExpandXipInplace: shouldExpandXipInplace) + return self.installArchivedXcode(xcode, at: url, to: destination, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: shouldExpandXipInplace) } .recover { error -> Promise in switch error { @@ -187,7 +195,7 @@ public final class XcodeInstaller { Current.logging.log(error.legibleLocalizedDescription.red) Current.logging.log("Removing damaged XIP and re-attempting installation.\n") try Current.files.removeItem(at: damagedXIPURL) - return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, shouldExpandXipInplace: shouldExpandXipInplace, attemptNumber: attemptNumber + 1) + return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: attemptNumber + 1, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: shouldExpandXipInplace) } } default: @@ -520,7 +528,7 @@ public final class XcodeInstaller { } } - public func installArchivedXcode(_ xcode: Xcode, at archiveURL: URL, to destination: Path, shouldExpandXipInplace: Bool) -> Promise { + public func installArchivedXcode(_ xcode: Xcode, at archiveURL: URL, to destination: Path, experimentalUnxip: Bool = false, shouldExpandXipInplace: Bool) -> Promise { let passwordInput = { Promise { seal in Current.logging.log("xcodes requires superuser privileges in order to finish installation.") @@ -533,7 +541,7 @@ public final class XcodeInstaller { let destinationURL = destination.join("Xcode-\(xcode.version.descriptionWithoutBuildMetadata).app").url switch archiveURL.pathExtension { case "xip": - return unarchiveAndMoveXIP(at: archiveURL, to: destinationURL, shouldExpandXipInplace: shouldExpandXipInplace).map { xcodeURL in + return unarchiveAndMoveXIP(at: archiveURL, to: destinationURL, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: shouldExpandXipInplace).map { xcodeURL in guard let path = Path(url: xcodeURL), Current.files.fileExists(atPath: path.string), @@ -717,10 +725,26 @@ public final class XcodeInstaller { } } - func unarchiveAndMoveXIP(at source: URL, to destination: URL, shouldExpandXipInplace: Bool) -> Promise { + func unarchiveAndMoveXIP(at source: URL, to destination: URL, experimentalUnxip: Bool, shouldExpandXipInplace: Bool) -> Promise { let xcodeExpansionDirectory = Current.files.xcodeExpansionDirectory(archiveURL: source, xcodeURL: destination, shouldExpandInplace: shouldExpandXipInplace) - return firstly { () -> Promise in - Current.logging.log(InstallationStep.unarchiving.description) + return firstly { () -> Promise in + Current.logging.log(InstallationStep.unarchiving(experimentalUnxip: experimentalUnxip).description) + + if experimentalUnxip, #available(macOS 11, *) { + return Promise { seal in + Task.detached { + let options = UnxipOptions(input: source, output: xcodeExpansionDirectory) + + do { + try await Unxip(options: options).run() + seal.resolve(.fulfilled(())) + } catch { + seal.reject(error) + } + } + } + } + return Current.shell.unxip(source, xcodeExpansionDirectory) .recover { (error) throws -> Promise in if case Process.PMKError.execution(_, _, let standardError) = error, @@ -729,8 +753,9 @@ public final class XcodeInstaller { } throw error } + .map { _ in () } } - .map { output -> URL in + .map { _ -> URL in Current.logging.log(InstallationStep.moving(destination: destination.path).description) let xcodeURL = xcodeExpansionDirectory.appendingPathComponent("Xcode.app") diff --git a/Sources/xcodes/main.swift b/Sources/xcodes/main.swift index 825edee..f37c646 100644 --- a/Sources/xcodes/main.swift +++ b/Sources/xcodes/main.swift @@ -183,6 +183,9 @@ struct Xcodes: ParsableCommand { @Flag(help: "Don't use aria2 to download Xcode, even if its available.") var noAria2: Bool = false + @Flag(help: "Use the experimental unxip functionality. May speed up unarchiving by up to 2-3x.") + var experimentalUnxip: Bool = false + @Option(help: "The directory to install Xcode into. Defaults to /Applications.", completion: .directory) var directory: String? @@ -221,7 +224,7 @@ struct Xcodes: ParsableCommand { let destination = getDirectory(possibleDirectory: directory) - installer.install(installation, dataSource: globalDataSource.dataSource, downloader: downloader, destination: destination, shouldExpandXipInplace: expandXipInplace) + installer.install(installation, dataSource: globalDataSource.dataSource, downloader: downloader, destination: destination, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: expandXipInplace) .done { Install.exit() } .catch { error in Install.processDownloadOrInstall(error: error) diff --git a/notarize.sh b/notarize.sh index b2d4e68..fa438f0 100755 --- a/notarize.sh +++ b/notarize.sh @@ -6,43 +6,14 @@ # # Adapted from https://github.com/keybase/client/blob/46f5df0aa64ff19198ba7b044bbb7cd907c0be9f/packaging/desktop/package_darwin.sh -username="$1" -password="$2" -asc_provider="$3" -file="$4" +file="$1" +team_id="$2" echo "Uploading to notarization service" -uuid=$(xcrun altool \ - --notarize-app \ - --primary-bundle-id "com.robotsandpencils.xcodes.zip" \ - --username "$username" \ - --password "$password" \ - --asc-provider "$asc_provider" \ - --file "$file" 2>&1 | \ - grep 'RequestUUID' | \ - awk '{ print $3 }') +result=$(xcrun notarytool submit "$file" \ + --keychain-profile "AC_PASSWORD" \ + --team-id "$team_id" \ + --wait) -echo "Successfully uploaded to notarization service, polling for result: $uuid" - -sleep 15 - while : - do - fullstatus=$(xcrun altool \ - --notarization-info "$uuid" \ - --username "$username" \ - --password "$password" \ - --asc-provider "$asc_provider" 2>&1) - status=$(echo "$fullstatus" | grep 'Status\:' | awk '{ print $2 }') - if [ "$status" = "success" ]; then - echo "Notarization success" - exit 0 - elif [ "$status" = "in" ]; then - echo "Notarization still in progress, sleeping for 15 seconds and trying again" - sleep 15 - else - echo "Notarization failed, full status below" - echo "$fullstatus" - exit 1 - fi - done +echo $result