From 58bea484c7ab50c356b5d9470df449a1774030a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Mar 2022 03:02:50 +0000 Subject: [PATCH 1/7] Bump actions/checkout from 2.4.0 to 3 Bumps [actions/checkout](https://github.com/actions/checkout) from 2.4.0 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2.4.0...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f4600a..65a669c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ jobs: test: runs-on: macOS-10.15 steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3 - name: Run tests env: DEVELOPER_DIR: /Applications/Xcode_12.3.app From 7a55373e1a8d737be55dc966602edf486c1b2a9b Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 2 Mar 2022 21:24:43 -0500 Subject: [PATCH 2/7] Add Unxip tool from saagarjha Source: https://github.com/saagarjha/unxip This mirrors this change in Xcodes.app: https://github.com/RobotsAndPencils/XcodesApp/pull/179 In my tests (M1 Max, 64GB RAM), unxipping is 3x faster than `/usr/bin/xip`. --- .../xcschemes/xcodes-Package.xcscheme | 28 + Package.swift | 1 + Sources/Unxip/Unxip.swift | 565 ++++++++++++++++++ 3 files changed, 594 insertions(+) create mode 100644 Sources/Unxip/Unxip.swift diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme index f2745c4..165d010 100644 --- a/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme +++ b/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme @@ -90,6 +90,34 @@ ReferencedContainer = "container:"> + + + + + + + + SubSequence { + let toOffset = toOffset ?? count + return self[index(startIndex, offsetBy: fromOffset).. SubSequence { + let base = index(startIndex, offsetBy: fromOffset) + return self[base.. { + let batchSize: Int + var operations = [@Sendable () async throws -> TaskResult]() + + var results: AsyncStream { + AsyncStream(bufferingPolicy: .bufferingOldest(batchSize)) { continuation in + Task { + try await withThrowingTaskGroup(of: (Int, TaskResult).self) { group in + var queueIndex = 0 + var dequeIndex = 0 + var pending = [Int: TaskResult]() + while dequeIndex < operations.count { + if queueIndex - dequeIndex < batchSize, + queueIndex < operations.count + { + let _queueIndex = queueIndex + group.addTask { + let queueIndex = _queueIndex + return await (queueIndex, try operations[queueIndex]()) + } + queueIndex += 1 + } else { + let (index, result) = try await group.next()! + pending[index] = result + if index == dequeIndex { + while let result = pending[dequeIndex] { + await continuation.yieldWithBackoff(result) + pending.removeValue(forKey: dequeIndex) + dequeIndex += 1 + } + } + } + } + continuation.finish() + } + } + } + } + + init(batchSize: Int = ProcessInfo.processInfo.activeProcessorCount) { + self.batchSize = batchSize + } + + mutating func addTask(operation: @escaping @Sendable () async throws -> TaskResult) { + operations.append(operation) + } + + mutating func addRunningTask(operation: @escaping @Sendable () async -> TaskResult) -> Task { + let task = Task { + await operation() + } + operations.append({ + await task.value + }) + return task + } +} + +final class Chunk: Sendable { + let buffer: UnsafeBufferPointer + let owned: Bool + + init(buffer: UnsafeBufferPointer, owned: Bool) { + self.buffer = buffer + self.owned = owned + } + + deinit { + if owned { + buffer.deallocate() + } + } +} + +struct File { + let dev: Int + let ino: Int + let mode: Int + let name: String + var data = [UnsafeBufferPointer]() + // For keeping the data alive + var chunks = [Chunk]() + + struct Identifier: Hashable { + let dev: Int + let ino: Int + } + + var identifier: Identifier { + Identifier(dev: dev, ino: ino) + } + + func writeCompressedIfPossible(usingDescriptor descriptor: CInt) async -> Bool { + let blockSize = 64 << 10 // LZFSE with 64K block size + var _data = [UInt8]() + _data.reserveCapacity(self.data.map(\.count).reduce(0, +)) + let data = self.data.reduce(into: _data, +=) + var compressionStream = ConcurrentStream<[UInt8]?>() + var position = data.startIndex + + while position < data.endIndex { + let _position = position + compressionStream.addTask { + try Task.checkCancellation() + let position = _position + let data = [UInt8](unsafeUninitializedCapacity: blockSize + blockSize / 16) { buffer, count in + data[position...size + let size = tableSize + chunks.map(\.count).reduce(0, +) + guard size < data.count else { + return false + } + + let buffer = [UInt8](unsafeUninitializedCapacity: size) { buffer, count in + var position = tableSize + + func writePosition(toTableIndex index: Int) { + precondition(position < UInt32.max) + for i in 0...size { + buffer[index * MemoryLayout.size + i] = UInt8(position >> (i * 8) & 0xff) + } + } + + writePosition(toTableIndex: 0) + for (index, chunk) in zip(1..., chunks) { + _ = UnsafeMutableBufferPointer(rebasing: buffer.suffix(from: position)).initialize(from: chunk) + position += chunk.count + writePosition(toTableIndex: index) + } + count = size + } + + let attribute = + "cmpf".utf8.reversed() // magic + + [0x0c, 0x00, 0x00, 0x00] // LZFSE, 64K chunks + + ([ + (data.count >> 0) & 0xff, + (data.count >> 8) & 0xff, + (data.count >> 16) & 0xff, + (data.count >> 24) & 0xff, + (data.count >> 32) & 0xff, + (data.count >> 40) & 0xff, + (data.count >> 48) & 0xff, + (data.count >> 56) & 0xff, + ].map(UInt8.init) as [UInt8]) + + guard fsetxattr(descriptor, "com.apple.decmpfs", attribute, attribute.count, 0, XATTR_SHOWCOMPRESSION) == 0 else { + return false + } + + let resourceForkDescriptor = open(name + _PATH_RSRCFORKSPEC, O_WRONLY | O_CREAT, 0o666) + guard resourceForkDescriptor >= 0 else { + return false + } + defer { + close(resourceForkDescriptor) + } + + var written: Int + repeat { + // TODO: handle partial writes smarter + written = pwrite(resourceForkDescriptor, buffer, buffer.count, 0) + guard written >= 0 else { + return false + } + } while written != buffer.count + + guard fchflags(descriptor, UInt32(UF_COMPRESSED)) == 0 else { + return false + } + + return true + } +} + +extension option { + init(name: StaticString, has_arg: CInt, flag: UnsafeMutablePointer?, val: StringLiteralType) { + let _option = name.withUTF8Buffer { + $0.withMemoryRebound(to: CChar.self) { + option(name: $0.baseAddress, has_arg: has_arg, flag: flag, val: CInt(UnicodeScalar(val)!.value)) + } + } + self = _option + } +} + +struct Options { + var input: URL + var output: URL? + var compress: Bool = true + + init() { + let options = [ + option(name: "compression-disable", has_arg: no_argument, flag: nil, val: "c"), + option(name: "help", has_arg: no_argument, flag: nil, val: "h"), + option(name: nil, has_arg: 0, flag: nil, val: 0), + ] + var result: CInt + repeat { + result = getopt_long(CommandLine.argc, CommandLine.unsafeArgv, "ch", options, nil) + if result < 0 { + break + } + switch UnicodeScalar(UInt32(result)) { + case "c": + compress = false + case "h": + Self.printUsage(nominally: true) + default: + Self.printUsage(nominally: false) + } + } while true + + let arguments = UnsafeBufferPointer(start: CommandLine.unsafeArgv + Int(optind), count: Int(CommandLine.argc - optind)).map { + String(cString: $0!) + } + + guard let input = arguments.first else { + Self.printUsage(nominally: false) + } + + self.input = URL(fileURLWithPath: input) + + guard let output = arguments.dropFirst().first else { + return + } + + self.output = URL(fileURLWithPath: output) + } + + static func printUsage(nominally: Bool) -> Never { + fputs( + """ + A fast Xcode unarchiver + + USAGE: unxip [options] [output] + + OPTIONS: + -c, --compression-disable Disable APFS compression of result. + -h, --help Print this help message. + """, nominally ? stdout : stderr) + exit(nominally ? EXIT_SUCCESS : EXIT_FAILURE) + } +} + +@main +struct Main { + static let options = Options() + + static func read(_ type: Integer.Type, from buffer: inout Buffer) -> Integer where Buffer.Element == UInt8, Buffer.SubSequence == Buffer { + defer { + buffer = buffer[fromOffset: MemoryLayout.size] + } + var result: Integer = 0 + var iterator = buffer.makeIterator() + for _ in 0...size { + result <<= 8 + result |= Integer(iterator.next()!) + } + return result + } + + static func chunks(from content: UnsafeBufferPointer) -> ConcurrentStream { + var remaining = content[fromOffset: 4] + let chunkSize = read(UInt64.self, from: &remaining) + var decompressedSize: UInt64 = 0 + + var chunkStream = ConcurrentStream() + + repeat { + decompressedSize = read(UInt64.self, from: &remaining) + let compressedSize = read(UInt64.self, from: &remaining) + let _remaining = remaining + let _decompressedSize = decompressedSize + + chunkStream.addTask { + let remaining = _remaining + let decompressedSize = _decompressedSize + + if compressedSize == chunkSize { + return Chunk(buffer: UnsafeBufferPointer(rebasing: remaining[fromOffset: 0, size: Int(compressedSize)]), owned: false) + } else { + let magic = [0xfd] + "7zX".utf8 + precondition(remaining.prefix(magic.count).elementsEqual(magic)) + let buffer = UnsafeMutableBufferPointer.allocate(capacity: Int(decompressedSize)) + precondition(compression_decode_buffer(buffer.baseAddress!, buffer.count, UnsafeBufferPointer(rebasing: remaining).baseAddress!, Int(compressedSize), nil, COMPRESSION_LZMA) == decompressedSize) + return Chunk(buffer: UnsafeBufferPointer(buffer), owned: true) + } + } + remaining = remaining[fromOffset: Int(compressedSize)] + } while decompressedSize == chunkSize + + return chunkStream + } + + static func files(in chunkStream: ChunkStream) -> AsyncStream where ChunkStream.Element == Chunk { + AsyncStream(bufferingPolicy: .bufferingOldest(ProcessInfo.processInfo.activeProcessorCount)) { continuation in + Task { + var iterator = chunkStream.makeAsyncIterator() + var chunk = try! await iterator.next()! + var position = 0 + + func read(size: Int) async -> [UInt8] { + var result = [UInt8]() + while result.count < size { + if position >= chunk.buffer.endIndex { + chunk = try! await iterator.next()! + position = 0 + } + result.append(chunk.buffer[chunk.buffer.startIndex + position]) + position += 1 + } + return result + } + + func readOctal(from bytes: [UInt8]) -> Int { + Int(String(data: Data(bytes), encoding: .utf8)!, radix: 8)! + } + + while true { + let magic = await read(size: 6) + // Yes, cpio.h really defines this global macro + precondition(magic.elementsEqual(MAGIC.utf8)) + let dev = readOctal(from: await read(size: 6)) + let ino = readOctal(from: await read(size: 6)) + let mode = readOctal(from: await read(size: 6)) + let _ = await read(size: 6) // uid + let _ = await read(size: 6) // gid + let _ = await read(size: 6) // nlink + let _ = await read(size: 6) // rdev + let _ = await read(size: 11) // mtime + let namesize = readOctal(from: await read(size: 6)) + var filesize = readOctal(from: await read(size: 11)) + let name = String(cString: await read(size: namesize)) + var file = File(dev: dev, ino: ino, mode: mode, name: name) + + while filesize > 0 { + if position >= chunk.buffer.endIndex { + chunk = try! await iterator.next()! + position = 0 + } + let size = min(filesize, chunk.buffer.endIndex - position) + file.chunks.append(chunk) + file.data.append(UnsafeBufferPointer(rebasing: chunk.buffer[fromOffset: position, size: size])) + filesize -= size + position += size + } + + guard file.name != "TRAILER!!!" else { + continuation.finish() + return + } + + await continuation.yieldWithBackoff(file) + } + } + } + } + + static func parseContent(_ content: UnsafeBufferPointer) async { + var taskStream = ConcurrentStream(batchSize: 64) // Worst case, should allow for files up to 64 * 16MB = 1GB + var hardlinks = [File.Identifier: (String, Task)]() + var directories = [Substring: Task]() + for await file in files(in: chunks(from: content).results) { + @Sendable + func warn(_ result: CInt, _ operation: String) { + if result != 0 { + perror("\(operation) \(file.name) failed") + } + } + + // The assumption is that all directories are provided without trailing slashes + func parentDirectory(of path: S) -> S.SubSequence { + return path[.. Task? { + directories[parentDirectory(of: file.name)] ?? directories[String(parentDirectory(of: file.name))[...]] + } + + @Sendable + func setStickyBit(on file: File) { + if file.mode & Int(C_ISVTX) != 0 { + warn(chmod(file.name, mode_t(file.mode)), "Setting sticky bit on") + } + } + + if file.name == "." { + continue + } + + if let (original, task) = hardlinks[file.identifier] { + _ = taskStream.addRunningTask { + await task.value + warn(link(original, file.name), "linking") + } + continue + } + + // The types we care about, anyways + let typeMask = Int(C_ISLNK | C_ISDIR | C_ISREG) + switch CInt(file.mode & typeMask) { + case C_ISLNK: + let task = parentDirectoryTask(for: file) + assert(task != nil, file.name) + _ = taskStream.addRunningTask { + warn(symlink(String(data: Data(file.data.map(Array.init).reduce([], +)), encoding: .utf8), file.name), "symlinking") + setStickyBit(on: file) + } + case C_ISDIR: + let task = parentDirectoryTask(for: file) + assert(task != nil || parentDirectory(of: file.name) == ".", file.name) + directories[file.name[...]] = taskStream.addRunningTask { + await task?.value + warn(mkdir(file.name, mode_t(file.mode & 0o777)), "creating directory at") + setStickyBit(on: file) + } + case C_ISREG: + let task = parentDirectoryTask(for: file) + assert(task != nil, file.name) + hardlinks[file.identifier] = ( + file.name, + taskStream.addRunningTask { + await task?.value + + let fd = open(file.name, O_CREAT | O_WRONLY, mode_t(file.mode & 0o777)) + if fd < 0 { + warn(fd, "creating file at") + return + } + defer { + warn(close(fd), "closing") + setStickyBit(on: file) + } + + if options.compress, + await file.writeCompressedIfPossible(usingDescriptor: fd) + { + return + } + + // pwritev requires the vector count to be positive + if file.data.count == 0 { + return + } + + var vector = file.data.map { + iovec(iov_base: UnsafeMutableRawPointer(mutating: $0.baseAddress), iov_len: $0.count) + } + let total = file.data.map(\.count).reduce(0, +) + var written = 0 + + repeat { + // TODO: handle partial writes smarter + written = pwritev(fd, &vector, CInt(vector.count), 0) + if written < 0 { + warn(-1, "writing chunk to") + break + } + } while written != total + } + ) + default: + fatalError("\(file.name) with \(file.mode) is a type that is unhandled") + } + } + + // Run through any stragglers + for await _ in taskStream.results { + } + } + + static func locateContent(in file: UnsafeBufferPointer) -> UnsafeBufferPointer { + precondition(file.starts(with: "xar!".utf8)) // magic + var header = file[4...] + let headerSize = read(UInt16.self, from: &header) + precondition(read(UInt16.self, from: &header) == 1) // version + let tocCompressedSize = read(UInt64.self, from: &header) + let tocDecompressedSize = read(UInt64.self, from: &header) + _ = read(UInt32.self, from: &header) // checksum + + let toc = [UInt8](unsafeUninitializedCapacity: Int(tocDecompressedSize)) { buffer, count in + let zlibSkip = 2 // Apple's decoder doesn't want to see CMF/FLG (see RFC 1950) + count = compression_decode_buffer(buffer.baseAddress!, Int(tocDecompressedSize), file.baseAddress! + Int(headerSize) + zlibSkip, Int(tocCompressedSize) - zlibSkip, nil, COMPRESSION_ZLIB) + precondition(count == Int(tocDecompressedSize)) + } + + let document = try! XMLDocument(data: Data(toc)) + let content = try! document.nodes(forXPath: "xar/toc/file").first { + try! $0.nodes(forXPath: "name").first!.stringValue! == "Content" + }! + let contentOffset = Int(try! content.nodes(forXPath: "data/offset").first!.stringValue!)! + let contentSize = Int(try! content.nodes(forXPath: "data/length").first!.stringValue!)! + let contentBase = Int(headerSize) + Int(tocCompressedSize) + contentOffset + + let slice = file[fromOffset: contentBase, size: contentSize] + precondition(slice.starts(with: "pbzx".utf8)) + return UnsafeBufferPointer(rebasing: slice) + } + + static func main() async throws { + let handle = try FileHandle(forReadingFrom: options.input) + try handle.seekToEnd() + let length = Int(try handle.offset()) + let file = UnsafeBufferPointer(start: mmap(nil, length, PROT_READ, MAP_PRIVATE, handle.fileDescriptor, 0).bindMemory(to: UInt8.self, capacity: length), count: length) + precondition(UnsafeMutableRawPointer(mutating: file.baseAddress) != MAP_FAILED) + defer { + munmap(UnsafeMutableRawPointer(mutating: file.baseAddress), length) + } + + if let output = options.output { + guard chdir(output.path) == 0 else { + fputs("Failed to access output directory at \(output.path): \(String(cString: strerror(errno)))", stderr) + exit(EXIT_FAILURE) + } + } + + await parseContent(locateContent(in: file)) + } +} From 5a18a6ec073d350a06d0199840f88166dd556d78 Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 2 Mar 2022 21:18:53 -0500 Subject: [PATCH 3/7] Adjust Unxip.swift for use in xcodes --- Sources/Unxip/Unxip.swift | 100 +++++++++----------------------------- 1 file changed, 22 insertions(+), 78 deletions(-) diff --git a/Sources/Unxip/Unxip.swift b/Sources/Unxip/Unxip.swift index 019e19a..7f3b9d5 100644 --- a/Sources/Unxip/Unxip.swift +++ b/Sources/Unxip/Unxip.swift @@ -1,6 +1,9 @@ import Compression import Foundation +// From: https://github.com/saagarjha/unxip +// License: GNU Lesser General Public License v3.0 + extension RandomAccessCollection { subscript(fromOffset fromOffset: Int = 0, toOffset toOffset: Int? = nil) -> SubSequence { let toOffset = toOffset ?? count @@ -13,6 +16,7 @@ extension RandomAccessCollection { } } +@available(macOS 11, *) extension AsyncStream.Continuation { func yieldWithBackoff(_ value: Element) async { let backoff: UInt64 = 1_000_000 @@ -22,6 +26,7 @@ extension AsyncStream.Continuation { } } +@available(macOS 11, *) struct ConcurrentStream { let batchSize: Int var operations = [@Sendable () async throws -> TaskResult]() @@ -96,6 +101,7 @@ final class Chunk: Sendable { } } +@available(macOS 11, *) struct File { let dev: Int let ino: Int @@ -217,81 +223,25 @@ struct File { } } -extension option { - init(name: StaticString, has_arg: CInt, flag: UnsafeMutablePointer?, val: StringLiteralType) { - let _option = name.withUTF8Buffer { - $0.withMemoryRebound(to: CChar.self) { - option(name: $0.baseAddress, has_arg: has_arg, flag: flag, val: CInt(UnicodeScalar(val)!.value)) - } - } - self = _option - } -} - -struct Options { +public struct UnxipOptions { var input: URL var output: URL? - var compress: Bool = true - - init() { - let options = [ - option(name: "compression-disable", has_arg: no_argument, flag: nil, val: "c"), - option(name: "help", has_arg: no_argument, flag: nil, val: "h"), - option(name: nil, has_arg: 0, flag: nil, val: 0), - ] - var result: CInt - repeat { - result = getopt_long(CommandLine.argc, CommandLine.unsafeArgv, "ch", options, nil) - if result < 0 { - break - } - switch UnicodeScalar(UInt32(result)) { - case "c": - compress = false - case "h": - Self.printUsage(nominally: true) - default: - Self.printUsage(nominally: false) - } - } while true - - let arguments = UnsafeBufferPointer(start: CommandLine.unsafeArgv + Int(optind), count: Int(CommandLine.argc - optind)).map { - String(cString: $0!) - } - guard let input = arguments.first else { - Self.printUsage(nominally: false) - } - - self.input = URL(fileURLWithPath: input) - - guard let output = arguments.dropFirst().first else { - return - } - - self.output = URL(fileURLWithPath: output) + public init(input: URL, output: URL) { + self.input = input + self.output = output } +} - static func printUsage(nominally: Bool) -> Never { - fputs( - """ - A fast Xcode unarchiver - - USAGE: unxip [options] [output] +@available(macOS 11, *) +public struct Unxip { + let options: UnxipOptions - OPTIONS: - -c, --compression-disable Disable APFS compression of result. - -h, --help Print this help message. - """, nominally ? stdout : stderr) - exit(nominally ? EXIT_SUCCESS : EXIT_FAILURE) + public init(options: UnxipOptions) { + self.options = options } -} - -@main -struct Main { - static let options = Options() - static func read(_ type: Integer.Type, from buffer: inout Buffer) -> Integer where Buffer.Element == UInt8, Buffer.SubSequence == Buffer { + func read(_ type: Integer.Type, from buffer: inout Buffer) -> Integer where Buffer.Element == UInt8, Buffer.SubSequence == Buffer { defer { buffer = buffer[fromOffset: MemoryLayout.size] } @@ -304,7 +254,7 @@ struct Main { return result } - static func chunks(from content: UnsafeBufferPointer) -> ConcurrentStream { + func chunks(from content: UnsafeBufferPointer) -> ConcurrentStream { var remaining = content[fromOffset: 4] let chunkSize = read(UInt64.self, from: &remaining) var decompressedSize: UInt64 = 0 @@ -337,7 +287,7 @@ struct Main { return chunkStream } - static func files(in chunkStream: ChunkStream) -> AsyncStream where ChunkStream.Element == Chunk { + func files(in chunkStream: ChunkStream) -> AsyncStream where ChunkStream.Element == Chunk { AsyncStream(bufferingPolicy: .bufferingOldest(ProcessInfo.processInfo.activeProcessorCount)) { continuation in Task { var iterator = chunkStream.makeAsyncIterator() @@ -401,7 +351,7 @@ struct Main { } } - static func parseContent(_ content: UnsafeBufferPointer) async { + func parseContent(_ content: UnsafeBufferPointer) async { var taskStream = ConcurrentStream(batchSize: 64) // Worst case, should allow for files up to 64 * 16MB = 1GB var hardlinks = [File.Identifier: (String, Task)]() var directories = [Substring: Task]() @@ -478,12 +428,6 @@ struct Main { setStickyBit(on: file) } - if options.compress, - await file.writeCompressedIfPossible(usingDescriptor: fd) - { - return - } - // pwritev requires the vector count to be positive if file.data.count == 0 { return @@ -515,7 +459,7 @@ struct Main { } } - static func locateContent(in file: UnsafeBufferPointer) -> UnsafeBufferPointer { + func locateContent(in file: UnsafeBufferPointer) -> UnsafeBufferPointer { precondition(file.starts(with: "xar!".utf8)) // magic var header = file[4...] let headerSize = read(UInt16.self, from: &header) @@ -543,7 +487,7 @@ struct Main { return UnsafeBufferPointer(rebasing: slice) } - static func main() async throws { + public func run() async throws { let handle = try FileHandle(forReadingFrom: options.input) try handle.seekToEnd() let length = Int(try handle.offset()) From 81758afc9d5b6b916dd04c340456dc38104d3dba Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 2 Mar 2022 17:05:21 -0500 Subject: [PATCH 4/7] Hook up experimental unxip flag --- Package.swift | 1 + Sources/XcodesKit/XcodeInstaller.swift | 39 +++++++++++++++++++------- Sources/xcodes/main.swift | 5 +++- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/Package.swift b/Package.swift index cdc0ba6..4a466f7 100644 --- a/Package.swift +++ b/Package.swift @@ -45,6 +45,7 @@ let package = Package( "PromiseKit", "PMKFoundation", "SwiftSoup", + "Unxip", "Version", .product(name: "XCModel", package: "XcodeReleases"), "Rainbow", diff --git a/Sources/XcodesKit/XcodeInstaller.swift b/Sources/XcodesKit/XcodeInstaller.swift index bbd872e..fc3171d 100644 --- a/Sources/XcodesKit/XcodeInstaller.swift +++ b/Sources/XcodesKit/XcodeInstaller.swift @@ -5,6 +5,7 @@ import AppleAPI import Version import LegibleError import Rainbow +import Unxip /// Downloads and installs Xcodes public final class XcodeInstaller { @@ -155,9 +156,9 @@ public final class XcodeInstaller { case aria2(Path) } - public func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path) -> Promise { + public func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, experimentalUnxip: Bool = false) -> Promise { return firstly { () -> Promise in - return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: 0) + return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: 0, experimentalUnxip: experimentalUnxip) } .done { xcode in Current.logging.log("\nXcode \(xcode.version.descriptionWithoutBuildMetadata) has been installed to \(xcode.path.string)".green) @@ -165,12 +166,12 @@ public final class XcodeInstaller { } } - private func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, attemptNumber: Int) -> Promise { + private func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, attemptNumber: Int, experimentalUnxip: Bool) -> Promise { return firstly { () -> Promise<(Xcode, URL)> in return self.getXcodeArchive(installationType, dataSource: dataSource, downloader: downloader, destination: destination, willInstall: true) } .then { xcode, url -> Promise in - return self.installArchivedXcode(xcode, at: url, to: destination) + return self.installArchivedXcode(xcode, at: url, to: destination, experimentalUnxip: experimentalUnxip) } .recover { error -> Promise in switch error { @@ -187,7 +188,7 @@ public final class XcodeInstaller { Current.logging.log(error.legibleLocalizedDescription.red) Current.logging.log("Removing damaged XIP and re-attempting installation.\n") try Current.files.removeItem(at: damagedXIPURL) - return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: attemptNumber + 1) + return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: attemptNumber + 1, experimentalUnxip: experimentalUnxip) } } default: @@ -520,7 +521,7 @@ public final class XcodeInstaller { } } - public func installArchivedXcode(_ xcode: Xcode, at archiveURL: URL, to destination: Path) -> Promise { + public func installArchivedXcode(_ xcode: Xcode, at archiveURL: URL, to destination: Path, experimentalUnxip: Bool = false) -> Promise { let passwordInput = { Promise { seal in Current.logging.log("xcodes requires superuser privileges in order to finish installation.") @@ -533,7 +534,7 @@ public final class XcodeInstaller { let destinationURL = destination.join("Xcode-\(xcode.version.descriptionWithoutBuildMetadata).app").url switch archiveURL.pathExtension { case "xip": - return unarchiveAndMoveXIP(at: archiveURL, to: destinationURL).map { xcodeURL in + return unarchiveAndMoveXIP(at: archiveURL, to: destinationURL, experimentalUnxip: experimentalUnxip).map { xcodeURL in guard let path = Path(url: xcodeURL), Current.files.fileExists(atPath: path.string), @@ -717,9 +718,26 @@ public final class XcodeInstaller { } } - func unarchiveAndMoveXIP(at source: URL, to destination: URL) -> Promise { - return firstly { () -> Promise in + func unarchiveAndMoveXIP(at source: URL, to destination: URL, experimentalUnxip: Bool) -> Promise { + return firstly { () -> Promise in Current.logging.log(InstallationStep.unarchiving.description) + + if experimentalUnxip, #available(macOS 11, *) { + return Promise { seal in + Task.detached { + let output = source.deletingLastPathComponent() + let options = UnxipOptions(input: source, output: output) + + do { + try await Unxip(options: options).run() + seal.resolve(.fulfilled(())) + } catch { + seal.reject(error) + } + } + } + } + return Current.shell.unxip(source) .recover { (error) throws -> Promise in if case Process.PMKError.execution(_, _, let standardError) = error, @@ -728,8 +746,9 @@ public final class XcodeInstaller { } throw error } + .map { _ in () } } - .map { output -> URL in + .map { _ -> URL in Current.logging.log(InstallationStep.moving(destination: destination.path).description) let xcodeURL = source.deletingLastPathComponent().appendingPathComponent("Xcode.app") diff --git a/Sources/xcodes/main.swift b/Sources/xcodes/main.swift index 5e37158..91f818e 100644 --- a/Sources/xcodes/main.swift +++ b/Sources/xcodes/main.swift @@ -183,6 +183,9 @@ struct Xcodes: ParsableCommand { @Flag(help: "Don't use aria2 to download Xcode, even if its available.") var noAria2: Bool = false + @Flag(help: "Use the experimental unxip functionality. May speed up unarchiving by up to 2-3x.") + var experimentalUnxip: Bool = false + @Option(help: "The directory to install Xcode into. Defaults to /Applications.", completion: .directory) var directory: String? @@ -218,7 +221,7 @@ struct Xcodes: ParsableCommand { let destination = getDirectory(possibleDirectory: directory) - installer.install(installation, dataSource: globalDataSource.dataSource, downloader: downloader, destination: destination) + installer.install(installation, dataSource: globalDataSource.dataSource, downloader: downloader, destination: destination, experimentalUnxip: experimentalUnxip) .done { Install.exit() } .catch { error in Install.processDownloadOrInstall(error: error) From 816130a2390d26c90f0444423d9320a0a14db67d Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 2 Mar 2022 22:07:53 -0500 Subject: [PATCH 5/7] Re-enable APFS compression of unxip output I removed this by mistake. --- Sources/Unxip/Unxip.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Sources/Unxip/Unxip.swift b/Sources/Unxip/Unxip.swift index 7f3b9d5..a078bd6 100644 --- a/Sources/Unxip/Unxip.swift +++ b/Sources/Unxip/Unxip.swift @@ -428,6 +428,10 @@ public struct Unxip { setStickyBit(on: file) } + if await file.writeCompressedIfPossible(usingDescriptor: fd) { + return + } + // pwritev requires the vector count to be positive if file.data.count == 0 { return From 0e679fd348bf49e8943efdd166e6856d1aff3d42 Mon Sep 17 00:00:00 2001 From: JP Simard Date: Tue, 8 Mar 2022 11:44:42 -0500 Subject: [PATCH 6/7] Log hints about experimental unxip flag when unarchiving Co-authored-by: Matt Kiazyk --- Sources/XcodesKit/XcodeInstaller.swift | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Sources/XcodesKit/XcodeInstaller.swift b/Sources/XcodesKit/XcodeInstaller.swift index fc3171d..81aba81 100644 --- a/Sources/XcodesKit/XcodeInstaller.swift +++ b/Sources/XcodesKit/XcodeInstaller.swift @@ -80,7 +80,7 @@ public final class XcodeInstaller { /// A numbered step enum InstallationStep: CustomStringConvertible { case downloading(version: String, progress: String?, willInstall: Bool) - case unarchiving + case unarchiving(experimentalUnxip: Bool) case moving(destination: String) case trashingArchive(archiveName: String) case checkingSecurity @@ -103,8 +103,15 @@ public final class XcodeInstaller { } else { return "Downloading Xcode \(version)" } - case .unarchiving: - return "Unarchiving Xcode (This can take a while)" + case .unarchiving(let experimentalUnxip): + let hint = experimentalUnxip ? + "Using experimental unxip. If you encounter any issues, remove the flag and try again" : + "Using regular unxip. Try passing `--experimental-unxip` for a faster unxip process" + return + """ + Unarchiving Xcode (This can take a while) + \(hint) + """ case .moving(let destination): return "Moving Xcode to \(destination)" case .trashingArchive(let archiveName): @@ -720,7 +727,7 @@ public final class XcodeInstaller { func unarchiveAndMoveXIP(at source: URL, to destination: URL, experimentalUnxip: Bool) -> Promise { return firstly { () -> Promise in - Current.logging.log(InstallationStep.unarchiving.description) + Current.logging.log(InstallationStep.unarchiving(experimentalUnxip: experimentalUnxip).description) if experimentalUnxip, #available(macOS 11, *) { return Promise { seal in From bfefa458388bab3da5352f8d24aeb2617fbe6d80 Mon Sep 17 00:00:00 2001 From: Matt Kiazyk Date: Wed, 9 Mar 2022 08:14:57 -0600 Subject: [PATCH 7/7] Bump version to 0.20.0 --- Makefile | 16 ++++++------ README.md | 15 +++++++++--- Sources/XcodesKit/Version.swift | 2 +- notarize.sh | 43 ++++++--------------------------- 4 files changed, 27 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index 18fe17b..d95f808 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ srcdir = Sources REPODIR = $(shell pwd) BUILDDIR = $(REPODIR)/.build SOURCES = $(wildcard $(srcdir)/**/*.swift) - +RELEASEBUILDDIR = $(BUILDDIR)/apple/Products/Release/xcodes .DEFAULT_GOAL = all .PHONY: all @@ -33,19 +33,19 @@ sign: xcodes --prefix com.robotsandpencils. \ --options runtime \ --timestamp \ - "$(BUILDDIR)/release/xcodes" + "$(RELEASEBUILDDIR)" .PHONY: zip zip: sign @rm xcodes.zip 2> /dev/null || true - @zip --junk-paths xcodes.zip "$(BUILDDIR)/release/xcodes" + @zip --junk-paths xcodes.zip "$(RELEASEBUILDDIR)" @open -R xcodes.zip # E.g. -# make notarize USERNAME="test@example.com" PASSWORD="@keychain:altool" ASC_PROVIDER=MyOrg +# make notarize TEAMID="ABCD123" .PHONY: notarize notarize: zip - ./notarize.sh "$(USERNAME)" "$(PASSWORD)" "$(ASC_PROVIDER)" xcodes.zip + ./notarize.sh xcodes.zip "$(TEAMID)" # E.g. # make bottle VERSION=0.4.0 @@ -54,7 +54,7 @@ bottle: sign @rm -r xcodes 2> /dev/null || true @rm *.tar.gz 2> /dev/null || true @mkdir -p xcodes/$(VERSION)/bin - @cp "$(BUILDDIR)/release/xcodes" xcodes/$(VERSION)/bin + @cp "$(RELEASEBUILDDIR)" xcodes/$(VERSION)/bin @tar -zcvf xcodes-$(VERSION).mojave.bottle.tar.gz -C "$(REPODIR)" xcodes shasum -a 256 xcodes-$(VERSION).mojave.bottle.tar.gz | cut -f1 -d' ' @open -R xcodes-$(VERSION).mojave.bottle.tar.gz @@ -62,7 +62,7 @@ bottle: sign .PHONY: install install: xcodes @install -d "$(bindir)" - @install "$(BUILDDIR)/release/xcodes" "$(bindir)" + @install "$(RELEASEBUILDDIR)" "$(bindir)" .PHONY: uninstall uninstall: @@ -70,7 +70,7 @@ uninstall: .PHONY: distclean distclean: - @rm -f $(BUILDDIR)/release + @rm -f $(RELEASEBUILDDIR) .PHONY: clean clean: distclean diff --git a/README.md b/README.md index 0b0ce97..c8b13b5 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ xcodes install 11.2 GM seed xcodes install 9.0 --path ~/Archive/Xcode_9.xip xcodes install --latest-prerelease xcodes install --latest --directory "/Volumes/Bag Of Holding/" +xcodes install --latest --experimental-unxip ``` You'll then be prompted to enter your Apple ID username and password. You can also provide these with the `XCODES_USERNAME` and `XCODES_PASSWORD` environment variables. @@ -106,6 +107,14 @@ Xcode will be installed to /Applications by default, but you can provide the pat - `version`: Print the version number of xcodes itself - `signout`: Clears the stored username and password +### Experimental Unxip - for faster unxipping + +Thanks to the amazing work by [saagarjhi](https://github.com/saagarjha/unxip) - Xcodes now includes the ability to unxip up to 70% faster on some systems. + +``` +xcodes install --latest --experimental-unxip +``` + ### Shell Completion Scripts xcodes can generate completion scripts which allow you to press the tab key on your keyboard to autocomplete commands and arguments when typing an xcodes command. The steps to install a completion script depend on the shell that you use. More information about installation instructions for different shells and the underlying implementation is available in the [swift-argument-parser repo](https://github.com/apple/swift-argument-parser/blob/main/Sources/ArgumentParser/Documentation.docc/Articles/InstallingCompletionScripts.md). @@ -123,7 +132,7 @@ xcodes --generate-completion-script > ~/.oh-my-zsh/completions/_xcodes ## Development -You'll need Xcode 12 in order to build and run xcodes. +You'll need Xcode 13 in order to build and run xcodes.
Using Xcode @@ -166,9 +175,7 @@ make bottle VERSION="$VERSION" # Notarize the release build # This can take a while make notarize \ - USERNAME="user@example.com" \ - PASSWORD="@keychain:ALTool Notarization" \ - ASC_PROVIDER="YourAppStoreConnectTeamName" + TEAMID="ABC123" # Push the new version bump commit and tag git push --follow-tags diff --git a/Sources/XcodesKit/Version.swift b/Sources/XcodesKit/Version.swift index 62fcb1b..e722549 100644 --- a/Sources/XcodesKit/Version.swift +++ b/Sources/XcodesKit/Version.swift @@ -1,3 +1,3 @@ import Version -public let version = Version("0.19.0")! +public let version = Version("0.20.0")! diff --git a/notarize.sh b/notarize.sh index b2d4e68..fa438f0 100755 --- a/notarize.sh +++ b/notarize.sh @@ -6,43 +6,14 @@ # # Adapted from https://github.com/keybase/client/blob/46f5df0aa64ff19198ba7b044bbb7cd907c0be9f/packaging/desktop/package_darwin.sh -username="$1" -password="$2" -asc_provider="$3" -file="$4" +file="$1" +team_id="$2" echo "Uploading to notarization service" -uuid=$(xcrun altool \ - --notarize-app \ - --primary-bundle-id "com.robotsandpencils.xcodes.zip" \ - --username "$username" \ - --password "$password" \ - --asc-provider "$asc_provider" \ - --file "$file" 2>&1 | \ - grep 'RequestUUID' | \ - awk '{ print $3 }') +result=$(xcrun notarytool submit "$file" \ + --keychain-profile "AC_PASSWORD" \ + --team-id "$team_id" \ + --wait) -echo "Successfully uploaded to notarization service, polling for result: $uuid" - -sleep 15 - while : - do - fullstatus=$(xcrun altool \ - --notarization-info "$uuid" \ - --username "$username" \ - --password "$password" \ - --asc-provider "$asc_provider" 2>&1) - status=$(echo "$fullstatus" | grep 'Status\:' | awk '{ print $2 }') - if [ "$status" = "success" ]; then - echo "Notarization success" - exit 0 - elif [ "$status" = "in" ]; then - echo "Notarization still in progress, sleeping for 15 seconds and trying again" - sleep 15 - else - echo "Notarization failed, full status below" - echo "$fullstatus" - exit 1 - fi - done +echo $result