fix: parse SLF strings with byte lengths

Package.swift

@@ -38,7 +38,10 @@ let package = Package(
         ),
         .testTarget(
             name: "XCLogParserTests",
-            dependencies: ["XCLogParser"]
+            dependencies: [
+                "XCLogParser",
+                .product(name: "Gzip", package: "GzipSwift")
+            ]
         ),
     ]
 

Package@swift-4.2.swift

@@ -31,7 +31,7 @@ let package = Package(
         ),
         .testTarget(
             name: "XCLogParserTests",
-            dependencies: ["XCLogParser"]
+            dependencies: ["XCLogParser", "Gzip"]
         ),
     ]
 

Package@swift-5.0.swift

@@ -34,7 +34,7 @@ let package = Package(
         ),
         .testTarget(
             name: "XCLogParserTests",
-            dependencies: ["XCLogParser"]
+            dependencies: ["XCLogParser", "Gzip"]
         ),
     ]
 

Sources/XCLogParser/lexer/Lexer.swift

@@ -264,16 +264,6 @@ public final class Lexer {
 
 private extension Scanner {
     var approximateLine: String {
-        let currentLocation = self.offset
-        let contentSize = self.string.count
-
-        let start = String.Index(compilerSafeOffset: currentLocation, in: self.string)
-        let endCount = contentSize - currentLocation > 21 ? currentLocation + 21 : contentSize - currentLocation
-        let end = String.Index(compilerSafeOffset: endCount, in: self.string)
-
-        if end <= start {
-            return String(self.string[start..<self.stringEndIndex])
-        }
-        return String(self.string[start..<end])
+        preview(count: 21)
     }
 }

Sources/XCLogParser/lexer/Scanner.swift

@@ -22,62 +22,71 @@ import Foundation
 final class Scanner {
 
     let string: String
+    private let bytes: [UInt8]
 
     private(set) var offset: Int
-    private(set) lazy var stringEndIndex: String.Index = self.string.endIndex
 
     var isAtEnd: Bool {
-        String.Index(compilerSafeOffset: self.offset, in: self.string) >= self.stringEndIndex
+        offset >= bytes.count
     }
 
     init(string: String) {
         self.string = string
+        self.bytes = Array(string.utf8)
         self.offset = 0
     }
 
     func scan(count: Int) -> String? {
-        let start = String.Index(compilerSafeOffset: self.offset, in: self.string)
         let endOffset = self.offset + count
 
-        guard endOffset <= self.string.utf16.count else { return nil }
-
-        let end = String.Index(compilerSafeOffset: endOffset, in: self.string)
-        let result = self.string[start..<end]
-
-        guard result.count == count else { return nil }
+        guard count >= 0,
+              endOffset <= bytes.count,
+              let result = String(bytes: bytes[offset..<endOffset], encoding: .utf8)
+        else { return nil }
 
         self.offset += count
 
-        return String(result)
+        return result
     }
 
     func scan(string value: String) -> Bool {
-        guard self.string.starts(with: value) else { return false }
+        let valueBytes = Array(value.utf8)
+        let endOffset = offset + valueBytes.count
+        guard endOffset <= bytes.count,
+              bytes[offset..<endOffset].elementsEqual(valueBytes)
+        else { return false }
 
-        self.offset += value.count
+        self.offset += valueBytes.count
         return true
     }
 
     func scanCharacters(from allowedCharacters: Set<Character>) -> String? {
-        var prefix: String = ""
-        var characterIndex = String.Index(compilerSafeOffset: self.offset, in: self.string)
-
-        while characterIndex < self.stringEndIndex {
-            let character = self.string[characterIndex]
+        let allowedBytes = Set(
+            allowedCharacters.compactMap { character -> UInt8? in
+                let characterBytes = Array(String(character).utf8)
+                return characterBytes.count == 1 ? characterBytes[0] : nil
+            }
+        )
+        var prefix: [UInt8] = []
 
-            guard allowedCharacters.contains(character) else {
+        while !isAtEnd {
+            guard allowedBytes.contains(bytes[offset]) else {
                 break
             }
 
-            prefix.append(character)
+            prefix.append(bytes[offset])
             self.offset += 1
-            characterIndex = String.Index(utf16Offset: self.offset, in: self.string)
         }
 
-        return prefix
+        return String(bytes: prefix, encoding: .utf8)
     }
 
     func moveOffset(by value: Int) {
         self.offset += value
     }
+
+    func preview(count: Int) -> String {
+        let endOffset = min(offset + count, bytes.count)
+        return String(decoding: bytes[offset..<endOffset], as: UTF8.self)
+    }
 }

Sources/XCLogParser/loglocation/LogLoader.swift

@@ -26,17 +26,7 @@ public struct LogLoader {
         do {
             let data = try Data(contentsOf: url)
             let unzipped = try data.gunzipped()
-            let string: String? = unzipped.withUnsafeBytes { pointer in
-                guard let charPointer = pointer
-                    .assumingMemoryBound(to: CChar.self)
-                    .baseAddress
-                else {
-                    return nil
-                }
-
-                return String(cString: charPointer, encoding: .ascii)
-            }
-            guard let contents = string else {
+            guard let contents = String(data: unzipped, encoding: .utf8) else {
                 throw LogError.readingFile(url.path)
             }
             return contents

Tests/XCLogParserTests/ActivityParserTests.swift

@@ -17,6 +17,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import Foundation
+import Gzip
 import XCTest
 @testable import XCLogParser
 
@@ -584,6 +586,30 @@ class ActivityParserTests: XCTestCase {
         XCTAssertEqual(10, parser.logVersion)
     }
 
+    func testParseActivityLogInURLPreservesUTF8ByteLengthStringsAndEmbeddedNullBytes() throws {
+        let title = "➜ Sources/Bundle+Locali🙂\u{0}zation"
+        var sectionTokens = IDEActivityLogSectionTokensWithoutAttachments
+        sectionTokens[2] = Token.string(title)
+        let tokens = [
+            Token.int(10),
+            Token.className("IDECommandLineBuildLog"),
+            Token.classNameRef("IDECommandLineBuildLog")
+        ] + sectionTokens
+        let directory = try TestUtils.createRandomTestDir()
+        defer { try? FileManager.default.removeItem(at: directory) }
+        let logURL = try writeActivityLog(contents: try slfContents(from: tokens), in: directory)
+
+        let activityLog = try parser.parseActivityLogInURL(
+            logURL,
+            redacted: false,
+            withoutBuildSpecificInformation: false
+        )
+
+        XCTAssertEqual(10, activityLog.version)
+        XCTAssertEqual(title, activityLog.mainSection.title)
+        XCTAssertEqual(10, parser.logVersion)
+    }
+
     func testParseDBGConsoleLog() throws {
         parser.logVersion = 11
         let tokens = DBGConsoleLogTokens
@@ -714,4 +740,42 @@ class ActivityParserTests: XCTestCase {
         XCTAssertEqual(metrics.taskCounters["SwiftDriver"]?["x"], 2)
     }
 
+    private func writeActivityLog(contents: String, in directory: URL) throws -> URL {
+        let url = directory.appendingPathComponent("test.xcactivitylog")
+        let data = contents.data(using: .utf8)!
+        try data.gzipped().write(to: url)
+        return url
+    }
+
+    private func slfContents(from tokens: [Token]) throws -> String {
+        var classNames: [String] = []
+        var contents = "SLF"
+        for token in tokens {
+            switch token {
+            case .int(let value):
+                contents += "\(value)#"
+            case .className(let value):
+                classNames.append(value)
+                contents += "\(value.utf8.count)%\(value)"
+            case .classNameRef(let value):
+                let index = try XCTUnwrap(
+                    classNames.firstIndex(of: value),
+                    "Class name \(value) must be emitted before it is referenced"
+                )
+                contents += "\(index + 1)@"
+            case .string(let value):
+                contents += "\(value.utf8.count)\"\(value)"
+            case .double(let value):
+                contents += "\(String(value.bitPattern.byteSwapped, radix: 16))^"
+            case .null:
+                contents += "-"
+            case .list(let value):
+                contents += "\(value)("
+            case .json(let value):
+                contents += "\(value.utf8.count)*\(value)"
+            }
+        }
+        return contents
+    }
+
 }

Tests/XCLogParserTests/LexerTests.swift

@@ -56,6 +56,15 @@ class LexerTests: XCTestCase {
         XCTAssertEqual(stringToken, Token.string("Xcode.IDEActivityLogDomainType.BuildLog"))
     }
 
+    func testTokenizeStringUsesUTF8ByteLength() throws {
+        let value = "➜ Sources/Bundle+Locali🙂"
+        let logContents = "SLF0#\(value.utf8.count)\"\(value)1#"
+
+        let tokens = try lexer.tokenize(contents: logContents, redacted: false, withoutBuildSpecificInformation: false)
+
+        XCTAssertEqual(tokens, [.int(0), .string(value), .int(1)])
+    }
+
     func testTokenizeDouble() throws {
         let logContents = "SLF09#21%IDEActivityLogSection1@39\"Xcode.IDEActivityLogDomainType.BuildLog356098f239dfc041^"
         let tokens = try lexer.tokenize(contents: logContents, redacted: false, withoutBuildSpecificInformation: false)

Tests/XCLogParserTests/LogLoaderTests.swift

@@ -0,0 +1,21 @@
+import Foundation
+import Gzip
+import XCTest
+@testable import XCLogParser
+
+class LogLoaderTests: XCTestCase {
+
+    func testLoadFromURLPreservesEmbeddedNullBytes() throws {
+        let directory = try TestUtils.createRandomTestDir()
+        defer { try? FileManager.default.removeItem(at: directory) }
+        let url = directory.appendingPathComponent("test.xcactivitylog")
+        let value = "Sources/Bundle+Locali\u{0}zation"
+        let contents = "SLF0#\(value.utf8.count)\"\(value)1#"
+        let data = contents.data(using: .utf8)!
+        try data.gzipped().write(to: url)
+
+        let loadedContents = try LogLoader().loadFromURL(url)
+
+        XCTAssertEqual(loadedContents, contents)
+    }
+}

Tests/XCLogParserTests/XCTestManifests.swift

@@ -7,6 +7,7 @@ extension ActivityParserTests {
     // to regenerate.
     static let __allTests__ActivityParserTests = [
         ("testParseActivityLog", testParseActivityLog),
+        ("testParseActivityLogInURLPreservesUTF8ByteLengthStringsAndEmbeddedNullBytes", testParseActivityLogInURLPreservesUTF8ByteLengthStringsAndEmbeddedNullBytes),
         ("testParseDBGConsoleLog", testParseDBGConsoleLog),
         ("testParseDVTTextDocumentLocation", testParseDVTTextDocumentLocation),
         ("testParseIBDocumentMemberLocation", testParseIBDocumentMemberLocation),
@@ -65,10 +66,20 @@ extension LexerTests {
         ("testTokenizeListNil", testTokenizeListNil),
         ("testTokenizeString", testTokenizeString),
         ("testTokenizeStringRedacted", testTokenizeStringRedacted),
+        ("testTokenizeStringUsesUTF8ByteLength", testTokenizeStringUsesUTF8ByteLength),
         ("testTokenizeStringWithTokenDelimiters", testTokenizeStringWithTokenDelimiters),
     ]
 }
 
+extension LogLoaderTests {
+    // DO NOT MODIFY: This is autogenerated, use:
+    //   `swift test --generate-linuxmain`
+    // to regenerate.
+    static let __allTests__LogLoaderTests = [
+        ("testLoadFromURLPreservesEmbeddedNullBytes", testLoadFromURLPreservesEmbeddedNullBytes),
+    ]
+}
+
 extension LogFinderTests {
     // DO NOT MODIFY: This is autogenerated, use:
     //   `swift test --generate-linuxmain`
@@ -138,6 +149,7 @@ public func __allTests() -> [XCTestCaseEntry] {
         testCase(IssuesReporterTests.__allTests__IssuesReporterTests),
         testCase(LexRedactorTests.__allTests__LexRedactorTests),
         testCase(LexerTests.__allTests__LexerTests),
+        testCase(LogLoaderTests.__allTests__LogLoaderTests),
         testCase(LogFinderTests.__allTests__LogFinderTests),
         testCase(LogManifestTests.__allTests__LogManifestTests),
         testCase(ParserTests.__allTests__ParserTests),