clang++ -DGGML_BUILD -DGGML_COMMIT=\"77d6ae4ac\" -DGGML_SCHED_MAX_COPIES=4 -DGGML_SHARED -DGGML_VERSION=\"0.9.7\" -D_GNU_SOURCE -D_XOPEN_SOURCE=600 -I/home/nick/workspace/llama.cpp/ggml/src/. -I/home/nick/workspace/llama.cpp/ggml/src/../include -std=gnu++17 -Xclang -dump-tokens -fsyntax-only /home/nick/workspace/llama.cpp/ggml/src/ggml.cpp

/usr/bin/c++ -E -DGGML_BUILD ... /home/nick/workspace/llama.cpp/ggml/src/ggml.cpp > processed_tokens.ii

/usr/bin/clang++ -Xclang -dump-tokens -fsyntax-only \
  -DGGML_BUILD -DGGML_COMMIT='"24d2ee052"' -DGGML_VERSION='"0.9.7"' \
  -DGGML_SCHED_MAX_COPIES=4 -DGGML_SHARED -D_GNU_SOURCE -D_XOPEN_SOURCE=600 \
  -I/home/nick/workspace/llama.cpp/ggml/src/. \
  -I/home/nick/workspace/llama.cpp/ggml/src/../include \
  -std=gnu++17 \
  /home/nick/workspace/llama.cpp/ggml/src/ggml.cpp 2>&1 | grep "ggml.cpp"

/usr/bin/clang++ -Xclang -dump-tokens -fsyntax-only \
  -DGGML_BUILD -DGGML_COMMIT='"24d2ee052"' -DGGML_VERSION='"0.9.7"' \
  -DGGML_SCHED_MAX_COPIES=4 -DGGML_SHARED -D_GNU_SOURCE -D_XOPEN_SOURCE=600 \
  -I/home/nick/workspace/llama.cpp/ggml/src/. \
  -I/home/nick/workspace/llama.cpp/ggml/src/../include \
  -std=gnu++17 \
  /home/nick/workspace/llama.cpp/ggml/src/ggml.cpp 2>&1 \
  | grep "ggml.cpp" \
  | awk '{print $1, $2}'

/usr/bin/clang++ -Xclang -dump-tokens -fsyntax-only \
  -DGGML_BUILD -DGGML_COMMIT='"24d2ee052"' -DGGML_VERSION='"0.9.7"' \
  -DGGML_SCHED_MAX_COPIES=4 -DGGML_SHARED -D_GNU_SOURCE -D_XOPEN_SOURCE=600 \
  -I/home/nick/workspace/llama.cpp/ggml/src/. \
  -I/home/nick/workspace/llama.cpp/ggml/src/../include \
  -std=gnu++17 \
  /home/nick/workspace/llama.cpp/ggml/src/ggml.cpp 2>&1 \
  | grep "ggml.cpp" \
  | awk '{
      token = $2; 
      gsub(/^\047|\047$/, "", token); # \047 是单引号的 ASCII 码，去掉首尾引号
      if (token == "") next;          # 过滤掉空的 token
      printf "%s ", token;            # 以空格分隔，不换行输出
    }'

| awk '{
    token = $2; gsub(/^\047|\047$/, "", token);
    if ($0 ~ /\[StartOfLine\]/) printf "\n"; 
    printf "%s ", token;
  }'

import json
import subprocess
import os
import re

# 配置文件
JSON_PATH = "compile_commands.json"
OUTPUT_DIR = "tokens_output"

if not os.path.exists(OUTPUT_DIR):
    os.makedirs(OUTPUT_DIR)

def process_command(item):
    source_file = item['file']
    # 只处理 C++ 文件 (可以根据需要增加 .cc 或 .cxx)
    if not source_file.endswith(('.cpp', '.cxx', '.cc')):
        return

    print(f"Processing: {os.path.basename(source_file)}")
    
    # 1. 解析原始命令
    cmd_parts = item['command'].split()
    
    # 2. 构造新的 Clang 指令
    # 我们需要插入 -Xclang -dump-tokens -fsyntax-only
    # 并移除 -o 和之后的目标文件
    new_cmd = [cmd_parts[0], "-Xclang", "-dump-tokens", "-fsyntax-only"]
    
    skip_next = False
    for i in range(1, len(cmd_parts)):
        if skip_next:
            skip_next = False
            continue
        if cmd_parts[i] == "-o":
            skip_next = True # 跳过 -o 及其后的文件名
            continue
        if cmd_parts[i] == "-c":
            continue # -c 留着或去掉都行，Clang 前端能识别
        new_cmd.append(cmd_parts[i])

    # 3. 执行指令并捕获输出
    try:
        # 运行并合并 stderr 到 stdout，因为 dump-tokens 输出到 stderr
        result = subprocess.run(
            new_cmd, 
            capture_output=True, 
            text=True, 
            cwd=item['directory']
        )
        
        raw_output = result.stdout + result.stderr
        
        # 4. 过滤与格式化 (纯 Token 流)
        tokens = []
        file_name_esc = re.escape(source_file)
        
        for line in raw_output.splitlines():
            # 只匹配当前源文件的 token，排除头文件
            if source_file in line:
                # 匹配格式: type 'value' [flags] Loc=...
                # 我们通过正则提取引号内的内容
                match = re.search(r"'(.*)'", line)
                if match:
                    tokens.append(match.group(1))
        
        # 5. 保存结果
        output_name = os.path.basename(source_file) + ".tokens"
        with open(os.path.join(OUTPUT_DIR, output_name), 'w') as f:
            f.write(" ".join(tokens))
            
    except Exception as e:
        print(f"Error processing {source_file}: {e}")

def main():
    with open(JSON_PATH, 'r') as f:
        data = json.load(f)
        for item in data:
            process_command(item)
    print(f"\nDone! All tokens are saved in '{OUTPUT_DIR}'")

if __name__ == "__main__":
    main()

import json
import os
from collections import Counter

class GrammarDiscoverer:
    def __init__(self, db_file="grammar_db.json"):
        self.db_file = db_file
        # grammar: {"NEW_SYMBOL_101": ["const", "char", "*"], ...}
        self.data = self.load_db()
        # 反向表，用于快速替换：{ ("const", "char", "*"): "NEW_SYMBOL_101" }
        self.rev_grammar = {tuple(v): k for k, v in self.data.get("grammar", {}).items()}
        self.next_id = self.data.get("next_id", 1000)

    def load_db(self):
        if os.path.exists(self.db_file):
            with open(self.db_file, 'r') as f: return json.load(f)
        return {"grammar": {}, "next_id": 1000}

    def save_db(self):
        self.data["next_id"] = self.next_id
        with open(self.db_file, 'w') as f: json.dump(self.data, f, indent=2)

    def simplify_stream(self, tokens):
        """利用已知语法表，对原始流进行坍缩(归纳再复用)"""
        changed = True
        while changed:
            changed = False
            for pattern, symbol in self.rev_grammar.items():
                n = len(pattern)
                i = 0
                new_tokens = []
                while i < len(tokens):
                    if tuple(tokens[i:i+n]) == pattern:
                        new_tokens.append(symbol)
                        i += n
                        changed = True
                    else:
                        new_tokens.append(tokens[i])
                        i += 1
                tokens = new_tokens
        return tokens

    def discover_new_patterns(self, tokens, min_freq=2):
        """核心：寻找新的 N-gram 并存入表"""
        # 我们从 2-gram 开始，因为所有长语法都是由 2-gram 逐步合并而来的
        # 就像 3NF：A -> BC, B -> de. 
        # 发现一个新的高频对 (A, B)，将其合并
        pairs = Counter(zip(tokens, tokens[1:]))
        if not pairs: return tokens, False

        most_common_pair, freq = pairs.most_common(1)[0]
        
        if freq >= min_freq:
            new_symbol = f"SYM_{self.next_id}"
            self.next_id += 1
            self.data["grammar"][new_symbol] = list(most_common_pair)
            self.rev_grammar[most_common_pair] = new_symbol
            print(f"发现新语法现象: {new_symbol} := {' '.join(most_common_pair)} (频率: {freq})")
            return self.simplify_stream(tokens), True
        
        return tokens, False

    def process_file(self, file_path):
        with open(file_path, 'r') as f:
            raw_tokens = f.read().split()
        
        # 1. 先用旧知识简化
        tokens = self.simplify_stream(raw_tokens)
        
        # 2. 循环发现新模式，直到没有新发现
        found = True
        while found:
            tokens, found = self.discover_new_patterns(tokens)
        
        self.save_db()
        return tokens

# 使用示例
discoverer = GrammarDiscoverer()
# 你可以不断加入新的 token 文件
final_stream = discoverer.process_file("ggml.cpp.tokens")

import json
import os
import sys
from collections import Counter

class GrammarDiscoverer:
    def __init__(self, db_file="grammar_db.json"):
        self.db_file = db_file
        self.data = self.load_db()
        self.rev_grammar = {tuple(v): k for k, v in self.data.get("grammar", {}).items()}
        self.next_id = self.data.get("next_id", 1000)

    def load_db(self):
        if os.path.exists(self.db_file):
            with open(self.db_file, 'r') as f: return json.load(f)
        return {"grammar": {}, "next_id": 1000}

    def save_db(self):
        self.data["next_id"] = self.next_id
        with open(self.db_file, 'w') as f: json.dump(self.data, f, indent=2)

    def expand_symbol(self, symbol):
        """递归展开符号，直到变成原始 Token"""
        if symbol not in self.data["grammar"]:
            return [symbol]
        
        res = []
        for part in self.data["grammar"][symbol]:
            res.extend(self.expand_symbol(part))
        return res

    def simplify_stream(self, tokens):
        changed = True
        while changed:
            changed = False
            for pattern, symbol in self.rev_grammar.items():
                n = len(pattern)
                i = 0
                new_tokens = []
                while i < len(tokens):
                    if tuple(tokens[i:i+n]) == pattern:
                        new_tokens.append(symbol)
                        i += n
                        changed = True
                    else:
                        new_tokens.append(tokens[i])
                        i += 1
                tokens = new_tokens
        return tokens

    def discover_new_patterns(self, tokens, min_freq=5): # 调高阈值以减少琐碎模式
        pairs = Counter(zip(tokens, tokens[1:]))
        if not pairs: return tokens, False

        most_common_pair, freq = pairs.most_common(1)[0]
        if freq >= min_freq:
            new_symbol = f"SYM_{self.next_id}"
            self.next_id += 1
            self.data["grammar"][new_symbol] = list(most_common_pair)
            self.rev_grammar[most_common_pair] = new_symbol
            
            # 打印直观的发现过程
            raw_repr = " ".join(self.expand_symbol(new_symbol))
            print(f"发现模式: {new_symbol: <9} := {raw_repr: <25} (出现 {freq} 次)")
            
            return self.simplify_stream(tokens), True
        return tokens, False

    def process_file(self, file_path):
        if not os.path.exists(file_path):
            print(f"错误: 文件 {file_path} 不存在")
            return
        
        with open(file_path, 'r') as f:
            raw_tokens = f.read().split()
        
        print(f"\n>>> 正在处理: {file_path}")
        tokens = self.simplify_stream(raw_tokens)
        
        found = True
        while found:
            tokens, found = self.discover_new_patterns(tokens)
        
        self.save_db()

    def print_final_grammar(self):
        print("\n" + "="*50)
        print("当前已归纳的语法表 (3NF 实例):")
        print("="*50)
        for sym in sorted(self.data["grammar"].keys()):
            parts = self.data["grammar"][sym]
            expanded = " ".join(self.expand_symbol(sym))
            # 显示 SYM_XXXX -> [A, B] (也就是实际的 Token 组合)
            print(f"{sym} -> {' + '.join(parts): <20} | 实际含义: {expanded}")

if __name__ == "__main__":
    # 从命令行获取文件名
    if len(sys.argv) < 2:
        print("用法: python3 script.py <token_file1> <token_file2> ...")
        sys.exit(1)

    discoverer = GrammarDiscoverer()
    
    for arg in sys.argv[1:]:
        discoverer.process_file(arg)
    
    discoverer.print_final_grammar()

python3 grammar_analyzer.py ggml.cpp.tokens other.tokens

import json, os, sys, re
from collections import Counter

class GrammarDiscoverer:
    def __init__(self, db_file="grammar_db.json"):
        self.db_file = db_file
        self.data = self.load_db()
        self.next_id = self.data.get("next_id", 1000)
        # 建立反向索引提高压缩速度
        self.refresh_rev_grammar()

    def load_db(self):
        if os.path.exists(self.db_file):
            with open(self.db_file, 'r') as f: return json.load(f)
        return {"grammar": {}, "next_id": 1000}

    def refresh_rev_grammar(self):
        # 按长度降序排列，优先匹配长序列（复用已知的高阶语法）
        self.rev_grammar = sorted(
            [(tuple(v), k) for k, v in self.data["grammar"].items()],
            key=lambda x: len(x[0]),
            reverse=True
        )

    def save_db(self):
        self.data["next_id"] = self.next_id
        with open(self.db_file, 'w') as f: json.dump(self.data, f, indent=2)

    def expand_symbol(self, symbol):
        if symbol not in self.data["grammar"]: return [symbol]
        res = []
        for part in self.data["grammar"][symbol]:
            res.extend(self.expand_symbol(part))
        return res

    def simplify_stream(self, tokens):
        """利用已有知识压缩 Token 流"""
        i = 0
        new_tokens = []
        while i < len(tokens):
            match_found = False
            for pattern, symbol in self.rev_grammar:
                n = len(pattern)
                if tuple(tokens[i:i+n]) == pattern:
                    new_tokens.append(symbol)
                    i += n
                    match_found = True
                    break
            if not match_found:
                new_tokens.append(tokens[i])
                i += 1
        return new_tokens

    def discover(self, tokens, max_n=10, min_freq=2):
        """逐级窗口扫描：从长到短寻找重复模式"""
        found_new = False
        # 从大的 N-gram 开始找，确保「长序列」优先被发现
        for n in range(max_n, 1, -1):
            grams = [tuple(tokens[i:i+n]) for i in range(len(tokens)-n+1)]
            counts = Counter(grams)
            
            # 找到当前长度下最频繁的序列
            common = [g for g, f in counts.items() if f >= min_freq]
            if not common: continue

            # 选一个最频繁的进行归纳
            best_gram = max(common, key=lambda g: counts[g])
            
            new_symbol = f"SYM_{self.next_id}"
            self.next_id += 1
            self.data["grammar"][new_symbol] = list(best_gram)
            self.refresh_rev_grammar()
            
            raw_repr = " ".join(self.expand_symbol(new_symbol))
            print(f"发现新语法 ({n}-gram): {new_symbol: <9} := {raw_repr: <30} (频率: {counts[best_gram]})")
            
            # 立即压缩当前的 token 流，并重新开始大窗口扫描
            return self.simplify_stream(tokens), True
        
        return tokens, False

    def process_file(self, file_path):
        if not os.path.exists(file_path): return
        with open(file_path, 'r') as f: tokens = f.read().split()
        
        print(f"\n>>> 正在处理: {file_path}")
        tokens = self.simplify_stream(tokens)
        
        while True:
            tokens, found = self.discover(tokens)
            if not found: break
        
        self.save_db()

if __name__ == "__main__":
    discoverer = GrammarDiscoverer()
    for arg in sys.argv[1:]:
        discoverer.process_file(arg)

import json, os, sys
from collections import Counter

class GrammarDiscoverer:
    def __init__(self, db_file="grammar_db.json"):
        self.db_file = db_file
        self.data = self.load_db()
        self.next_id = self.data.get("next_id", 1000)
        self.grammar = self.data.get("grammar", {})

    def load_db(self):
        if os.path.exists(self.db_file):
            with open(self.db_file, 'r') as f: return json.load(f)
        return {"grammar": {}, "next_id": 1000}

    def save_db(self):
        self.data["grammar"] = self.grammar
        self.data["next_id"] = self.next_id
        with open(self.db_file, 'w') as f: json.dump(self.data, f, indent=2)

    def expand(self, sym):
        if sym not in self.grammar: return [sym]
        res = []
        for p in self.grammar[sym]: res.extend(self.expand(p))
        return res

    def get_token_stream(self, tokens):
        """核心：贪婪压缩。用已有的最长规则去坍缩当前的流"""
        # 按规则展开后的长度降序排列，保证优先匹配复杂语法
        sorted_rules = sorted(self.grammar.items(), key=lambda x: len(self.expand(x[0])), reverse=True)
        
        i = 0
        new_stream = []
        while i < len(tokens):
            matched = False
            for sym, pattern in sorted_rules:
                plen = len(pattern)
                if tokens[i:i+plen] == pattern:
                    new_stream.append(sym)
                    i += plen
                    matched = True
                    break
            if not matched:
                new_stream.append(tokens[i])
                i += 1
        return new_stream

    def discover(self, tokens, min_freq=2):
        """
        不再盲目增加 N-gram 长度，而是寻找当前流中频率最高的 2-Gram 组合。
        因为流已经经过了 get_token_stream 的坍缩，这里的 2-Gram 可能代表了：
        1. Token + Token (基础)
        2. SYM + Token  (扩展)
        3. SYM + SYM    (递归或复合结构)
        """
        pairs = Counter(zip(tokens, tokens[1:]))
        if not pairs: return tokens, False

        # 过滤掉频率不足的
        candidates = {p: f for p, f in pairs.items() if f >= min_freq}
        if not candidates: return tokens, False

        # 策略：选择能带来最大压缩率的组合
        # 或者选择最频繁的
        best_pair = max(candidates, key=candidates.get)
        
        # 检查是否构成了自引用(简单递归初探)
        # 如果 A -> A B 或 A -> B A，这在 3NF 中是典型的递归
        
        new_sym = f"SYM_{self.next_id}"
        self.next_id += 1
        self.grammar[new_sym] = list(best_pair)
        
        raw_repr = " ".join(self.expand(new_sym))
        print(f"归纳规则: {new_sym: <9} := {' + '.join(best_pair): <25} | 实际语义: {raw_repr}")
        
        # 归纳后立即重新坍缩流
        return self.get_token_stream(tokens), True

    def process(self, file_path):
        if not os.path.exists(file_path): return
        with open(file_path, 'r') as f: tokens = f.read().split()
        
        # 1. 用已有知识预压缩
        tokens = self.get_token_stream(tokens)
        
        # 2. 迭代归纳
        while True:
            tokens, found = self.discover(tokens)
            if not found: break
        
        self.save_db()

if __name__ == "__main__":
    gd = GrammarDiscoverer()
    for f in sys.argv[1:]:
        gd.process(f)

import json, os, sys
from collections import Counter

class GrammarDiscovery:
    def __init__(self, db_file="grammar_v2.json"):
        self.db_file = db_file
        self.grammar = {} # 存储发现的所有合法序列
        self.recursive_patterns = [] # 存储发现的递归现象

    def get_ngrams(self, tokens, n):
        """穷举所有的 n-gram"""
        return [tuple(tokens[i:i+n]) for i in range(len(tokens)-n+1)]

    def is_recursive(self, sequence, known_grammars):
        """
        判断一个长序列是否包含递归。
        例如：if ( if ( A ) ) 包含了 if ( A )
        """
        seq_str = " ".join(sequence)
        for known in known_grammars:
            known_str = " ".join(known)
            if known_str in seq_str and known_str != seq_str:
                return True, known
        return False, None

    def run_discovery(self, tokens, start_n=2, max_n=20):
        current_tokens = tokens
        
        for n in range(start_n, max_n + 1):
            print(f"\n--- 正在扫描 {n}-gram 阶段 ---")
            ngrams = self.get_ngrams(current_tokens, n)
            unique_ngrams = set(ngrams)
            
            new_discoveries = 0
            for gram in unique_ngrams:
                gram_key = " ".join(gram)
                
                if gram_key not in self.grammar:
                    # 检查递归：这个 n-gram 是否由之前发现的语法现象嵌套而成？
                    found_rec, sub_pattern = self.is_recursive(gram, self.grammar.keys())
                    
                    self.grammar[gram_key] = {
                        "n": n,
                        "is_recursive": found_rec,
                        "sub_structure": " ".join(sub_pattern) if found_rec else None
                    }
                    new_discoveries += 1
                    
                    if found_rec:
                        print(f"[递归发现] {gram_key} (包含了已知结构: {self.grammar[gram_key]['sub_structure']})")
            
            print(f"本轮新增合法语法现象: {new_discoveries} 条")
            
            # 如果连续两轮没有新发现，或者新发现全是递归，说明语法特征已饱和
            if new_discoveries == 0:
                print(f"语法在 N={n} 处达到饱和，停止扫描。")
                break

    def save(self):
        with open(self.db_file, 'w') as f:
            json.dump(self.grammar, f, indent=2)

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python script.py <token_file>")
        sys.exit(1)

    with open(sys.argv[1], 'r') as f:
        tokens = f.read().split()

    discovery = GrammarDiscovery()
    discovery.run_discovery(tokens)
    discovery.save()

# 在处理 Clang dump-tokens 的输出时进行映射
def canonicalize(token_type, token_value):
    # Clang 的 token_type 包含 'identifier', 'numeric_constant', 'string_literal' 等
    if token_type == 'identifier':
        # 排除 C++ 关键字，只抽象普通变量/函数名
        if is_cpp_keyword(token_value): 
            return token_value
        return "<id>"
    if token_type == 'numeric_constant':
        return "<num>"
    if token_type == 'string_literal':
        return "<str>"
    return token_value # 符号和关键字保持原样

/usr/bin/clang++ -Xclang -dump-tokens -fsyntax-only \
  -DGGML_BUILD -DGGML_COMMIT='"24d2ee052"' -DGGML_VERSION='"0.9.7"' \
  -DGGML_SCHED_MAX_COPIES=4 -DGGML_SHARED -D_GNU_SOURCE -D_XOPEN_SOURCE=600 \
  -I/home/nick/workspace/llama.cpp/ggml/src/. \
  -I/home/nick/workspace/llama.cpp/ggml/src/../include \
  -std=gnu++17 \
  /home/nick/workspace/llama.cpp/ggml/src/ggml.cpp 2>&1 \
  | grep "ggml.cpp" \
  | awk '{
      # $1 是 Token 类型 (例如 identifier, l_paren, kw_if)
      token_type = $1;
      
      # 如果你想要更精细的抽象，可以在这里对类型进行简单的映射
      # 例如：numeric_constant -> literal, string_literal -> literal
      if (token_type ~ /_constant$|literal$/) token_type = "literal";
      
      printf "%s ", token_type;
    }'