vendor: amis 6.13.0 base (fork root)

This commit is contained in:
2026-06-16 22:14:57 +08:00
commit 8c8e4247a9
3840 changed files with 2467374 additions and 0 deletions

View File

@@ -0,0 +1 @@
base

7
scripts/bot/.gitignore vendored Normal file
View File

@@ -0,0 +1,7 @@
db
__pycache__
text.json
embedding.json
.env
m3e-base
flagged

10
scripts/bot/Dockerfile Normal file
View File

@@ -0,0 +1,10 @@
FROM amis_bot_base:v2
WORKDIR /app
EXPOSE 7860
COPY . .
CMD [ "python", "./gui.py" ]

3
scripts/bot/README.md Normal file
View File

@@ -0,0 +1,3 @@
# 基于大模型的 QA 机器人示例
需要有文心千帆 AK/SK

14
scripts/bot/app.py Normal file
View File

@@ -0,0 +1,14 @@
from fastapi import FastAPI
app = FastAPI()
@app.get("/")
def read_root():
return "use /query/{query}"
@app.get("/amis-doc-query/{query}")
def read_item(query: str):
return {"item_id": "d"}

View File

@@ -0,0 +1,8 @@
FROM python:3.11.4-bookworm
WORKDIR /app
COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
COPY m3e-base m3e-base

View File

@@ -0,0 +1,7 @@
sentence_transformers
fastapi
markdown
python-dotenv
gradio
requests
chromadb

13
scripts/bot/embedding.py Normal file
View File

@@ -0,0 +1,13 @@
import os
from sentence_transformers import SentenceTransformer, util
model_name = 'moka-ai/m3e-base'
if os.getenv('EMBEDDING_MODEL') != None:
model_name = os.getenv('EMBEDDING_MODEL')
model = SentenceTransformer(model_name)
def get_embedding(text):
return model.encode(text)

View File

@@ -0,0 +1,72 @@
import sys
import os
import glob
import uuid
import json
from embedding import get_embedding
from split_markdown import split_markdown
from vector_store import get_client
chroma_client = get_client()
# 每次执行都会清理避免重复
chroma_client.reset()
collection = chroma_client.create_collection(name="amis")
doc_dir = sys.argv[1]
# 存储所有文本段用于大模型生成
text_blocks_by_id = {}
# embedding 缓存,虽然目前速度很快,但后续如果换成网络请求会比较慢
embedding_cache = {}
embedding_cache_file = os.path.join(
os.path.dirname(__file__), 'embedding.json')
if os.path.exists(embedding_cache_file):
with open(embedding_cache_file, 'rb') as f:
embedding_cache = json.load(f)
def get_embedding_with_cache(text):
if text in embedding_cache:
return embedding_cache[text]
else:
embedding = get_embedding(text).tolist()
embedding_cache[text] = embedding
return embedding
for filename in glob.iglob(doc_dir + '**/*.md', recursive=True):
with open(filename) as f:
content = f.read()
md_blocks = split_markdown(content, filename)
embeddings = []
documents = []
metadatas = []
ids = []
for block in md_blocks:
block_id = uuid.uuid4().hex
text_blocks_by_id[block_id] = block
text_blocks = block.get_text_blocks()
index = 1
for text_block in text_blocks:
embeddings.append(get_embedding_with_cache(text_block))
documents.append(text_block)
ids.append(block_id + "_" + str(index))
metadatas.append({"source": block.file_name})
index += 1
collection.add(
embeddings=embeddings,
documents=documents,
metadatas=metadatas,
ids=ids
)
with open(os.path.join(os.path.dirname(__file__), 'text.json'), 'w') as f:
json.dump(text_blocks_by_id, f)
with open(embedding_cache_file, 'w') as f:
json.dump(embedding_cache, f)

111
scripts/bot/gui.py Normal file
View File

@@ -0,0 +1,111 @@
from vector_store import get_client
from split_markdown import split_markdown
from embedding import get_embedding
import gradio as gr
import os
import json
from llm.wenxin import Wenxin, ModelName
from dotenv import load_dotenv
load_dotenv()
chroma_client = get_client()
collection = chroma_client.get_collection(name="amis")
wenxin = Wenxin()
text_blocks_by_id = {}
with open(os.path.join(os.path.dirname(__file__), 'text.json'), 'rb') as f:
text_blocks_by_id = json.load(f)
def get_prompt(context, query):
return f"""
请只根据下面的资料回答问题,如果无法根据这些资料回答,回答“找不到相关答案”:
资料:
{context}
问题是:{query}
回答:"""
def get_context(search_result, include_code=True, max_length=1024):
context = ""
doc_ids = []
for doc_id in search_result['ids'][0]:
doc_id = doc_id.split("_")[0]
if doc_id not in doc_ids:
doc_ids.append(doc_id)
for doc_id in doc_ids:
markdown_block = text_blocks_by_id[doc_id]
block_text = markdown_block.gen_text(512, include_code)
if (len(context) + len(block_text)) < max_length:
context += block_text + "\n\n"
return context
query = gr.Textbox(label="问题")
include_code = gr.Checkbox(value=True, label="提示词中是否要包含 amis schema",
info="包含的好处是大模型会返回 json,但也会导致内容太长,只能提供少量段落给大模型,导致错过重要资料")
n_result = gr.Number(
value=10, precision=0, label="向量搜索查询返回个数")
bot_result = gr.Textbox(label="文心的回答")
bot_turbo_result = gr.Textbox(label="文心 Turbo 的回答")
booomz_result = gr.Textbox(label="开源 BLOOMZ 的回答")
prompt = gr.Textbox(label="提示词")
vector_search_result = gr.Dataframe(
label="向量相关搜索结果,这个结果只是为了辅助调试,确认是因为没找到相关内容还是大模型没能理解",
headers=["相关段落", "所属文档"],
datatype=["str", "str"],
col_count=(2, "dynamic"),
wrap=True
)
def amis_search(query, n_result=10, include_code=True):
if query.strip() == "":
return "必须有输入", "", "", []
search_result = collection.query(
query_embeddings=get_embedding(query).tolist(),
n_results=n_result
)
context = get_context(search_result, include_code)
if (context == ""):
return "检索不到相关内容", "", "", []
prompt = get_prompt(context, query)
bot_result = wenxin.generate(prompt, ModelName.ERNIE_BOT)
# bloomz_result = wenxin.generate(prompt, ModelName.BLOOMZ_7B)
markdown_blocks = []
index = 0
for doc in search_result['documents'][0]:
markdown_block = []
markdown_block.append(doc)
if index < len(search_result['metadatas'][0]):
source = search_result['metadatas'][0][index]['source'].replace(
'docs/zh-CN/', '')
markdown_block.append(
source)
else:
print("index out of range", doc)
markdown_blocks.append(markdown_block)
index += 1
return bot_result, prompt, markdown_blocks
demo = gr.Interface(amis_search, title="amis 文档问答机器人", inputs=[
query, n_result, include_code], outputs=[bot_result, prompt, vector_search_result])
if __name__ == '__main__':
demo.queue(concurrency_count=10).launch(share=False, server_name="0.0.0.0")

7
scripts/bot/llm/base.py Normal file
View File

@@ -0,0 +1,7 @@
from abc import ABC, abstractmethod
class LLM(ABC):
@abstractmethod
def generate(self, prompt: str) -> str:
pass

71
scripts/bot/llm/wenxin.py Normal file
View File

@@ -0,0 +1,71 @@
from .base import LLM
import requests
from functools import lru_cache
import time
import os
import json
from enum import Enum
from requests.exceptions import HTTPError
base_url = 'https://aip.baidubce.com'
class ModelName(Enum):
ERNIE_BOT = 1
ERNIE_BOT_TURBO = 2
BLOOMZ_7B = 3
def get_ttl_hash(seconds=3600):
"""缓存一小时"""
return round(time.time() / seconds)
@lru_cache(maxsize=1)
def get_token(ttl_hash=None):
"""
根据 ak/sk 获取 access_token
"""
del ttl_hash
ak = os.getenv('WENXIN_AK')
sk = os.getenv('WENXIN_SK')
url = f'{base_url}/oauth/2.0/token?grant_type=client_credentials&client_id={ak}&client_secret={sk}'
response = requests.get(url)
response.raise_for_status()
return response.json()['access_token']
def query(query: str, token: str, model_name: ModelName = ModelName.ERNIE_BOT_TURBO):
"""
参考文档
https://cloud.baidu.com/doc/WENXINWORKSHOP/s/jlil56u11
"""
# 目前暂时写死
user_id = os.getenv('WENXIN_USER_ID')
model_path = "eb-instant"
if model_name == ModelName.BLOOMZ_7B:
model_path = "bloomz_7b1"
elif model_name == ModelName.ERNIE_BOT:
model_path = "completions"
url = f'{base_url}/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/{model_path}?access_token={token}&user_id={user_id}'
messages = [{'role': 'user', 'content': query}]
payload = {
'messages': messages
}
headers = {
'content-type': 'application/json',
}
response = requests.post(url, data=json.dumps(payload), headers=headers)
response.raise_for_status()
return response.json()["result"]
class Wenxin(LLM):
"""文心千帆大模型"""
def generate(self, prompt: str, model_name: ModelName = ModelName.ERNIE_BOT_TURBO) -> str:
try:
return query(prompt, get_token(ttl_hash=get_ttl_hash()), model_name)
except HTTPError as e:
return f'HTTPError: {e}'

View File

@@ -0,0 +1,54 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/nwind/miniconda3/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n",
"No sentence-transformers model found with name /Volumes/WD/ai/models/m3e-base. Creating a new one with MEAN pooling.\n"
]
}
],
"source": [
"from dotenv import load_dotenv\n",
"\n",
"load_dotenv()\n",
"\n",
"from embedding import get_embedding\n",
"from split_markdown import split_markdown\n",
"from vector_store import get_client\n",
"\n",
"chroma_client = get_client()\n",
"collection = chroma_client.get_collection(name=\"amis\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}

View File

@@ -0,0 +1,210 @@
import sys
import re
from enum import Enum
from dataclasses import dataclass, field
from markdown import Markdown
from io import StringIO
def unmark_element(element, stream=None):
if stream is None:
stream = StringIO()
if element.text:
stream.write(element.text)
for sub in element:
unmark_element(sub, stream)
if element.tail:
stream.write(element.tail)
return stream.getvalue()
# patching Markdown
Markdown.output_formats["plain"] = unmark_element
__md = Markdown(output_format="plain")
__md.stripTopLevelTags = False
def unmark(text):
"""去掉文本样式,参考 https://stackoverflow.com/questions/761824/python-how-to-convert-markdown-formatted-text-to-text"""
return __md.convert(text)
class ContentType(Enum):
Text = 1
Code = 2
# 最长段落
LONG_CONTENT_LENGTH = 20
@dataclass
class BlockContent:
"""文本段中的文本或代码"""
type: ContentType
text: str
@dataclass
class MarkdownBlock:
"""文档段,这个是给大模型上下文的最小单位"""
# 文件名
file_name: str
# 文件标题
title: str = ""
# 二级或三级标题
header: str = ""
# 内容,可能是文本或代码段
content: list[ContentType] = field(default_factory=list)
def gen_text(self, max_length: int = 500, include_code=True) -> str:
""""输出文本"""
current_length = 0
output = self.header + "\n\n" if self.header else ""
for para in self.content:
content = para.text
# 超过长度限制了就中断,这里其实没考虑代码段 ``` 多出来的 10 个字符
if current_length + len(content) > max_length:
break
if para.type == ContentType.Code and include_code:
output += f"\n```\n{content}\n```\n"
else:
output += content + "\n"
current_length += len(content)
return output
def get_text_blocks(self) -> list[str]:
"""获取用于生成嵌入的文本段落列表"""
blocks: list[str] = []
header = self.header.replace("#", "") if self.header else ""
if header != "":
if len(header) < 4:
blocks.append(self.title + header)
else:
blocks.append(header)
all_text = ""
for para in self.content:
if para.type == ContentType.Text:
# 去掉各种样式及图片避免影响
text = unmark(para.text)
all_text += text
blocks.append(self.title + header + text)
blocks.append(text)
# 对于太长的段落,拆分一下
if len(text) > LONG_CONTENT_LENGTH:
for line in text.split(","):
blocks.append(line)
if len(all_text) < LONG_CONTENT_LENGTH:
blocks.append(header + all_text)
# 删掉重复的和避免空字符
output_blocks = set()
for block in blocks:
block = block.strip()
if block != "" and block not in output_blocks:
output_blocks.add(block)
return list(output_blocks)
def split_markdown(markdown_text: str, file_name: str) -> list[MarkdownBlock]:
"""
拆分 Markdown 文档为段落
"""
markdown_text = markdown_text.replace("\r\n", "\n").replace("\r", "\n")
# 文档标题
title = ""
lines = markdown_text.split("\n")
# markdown 段落
blocks: list[MarkdownBlock] = []
# 当前二级标题
current_header = None
current_content: list[BlockContent] = []
# 代码需要合并到一起,所以先收集
current_code: list[str] = []
# 是否在代码快中
in_code_block = False
# 文档元数据
in_meta = False
for line in lines:
# 处理文档元数据
if line.startswith("---"):
in_meta = not in_meta
continue
if in_meta and ":" in line:
key, value = line.split(":")
if key == "title":
title = value.strip()
continue
# 这是版本说明,没什么用
if line.startswith("> ") and "以上版本" in line:
continue
if line.startswith(">"):
line = line.replace(">", "")
if line.strip() == "":
continue
header_match = re.match(r"^#+\s", line)
# 匹配到了标题
if header_match:
# 如果之前有标题,那么这就是新的一段
if current_header is not None:
# 至少要有内容或者代码块
if len(current_content) > 0:
blocks.append(MarkdownBlock(file_name, title,
current_header, current_content))
current_content = []
current_code = []
# 开启新段落解析
current_header = line
else:
# 说明是刚开始的文本,没有标题
if current_header is None:
current_content.append(BlockContent(ContentType.Text, line))
blocks.append(MarkdownBlock(file_name, title,
current_header, current_content))
current_content = []
else:
# 说明是代码块
if line.startswith("```"):
in_code_block = not in_code_block
if not in_code_block:
current_content.append(BlockContent(
ContentType.Code, "\n".join(current_code)))
current_code = []
else:
if in_code_block:
current_code.append(line)
else:
current_content.append(
BlockContent(ContentType.Text, line))
if len(current_content) > 0 or len(current_code) > 0:
blocks.append(MarkdownBlock(file_name, title,
current_header, current_content))
return blocks
def test(file_name: str):
with open(file_name) as f:
content = f.read()
blocks = split_markdown(content, file_name)
for block in blocks:
print(block.getText())
if __name__ == '__main__':
test(sys.argv[1])

View File

@@ -0,0 +1,15 @@
import chromadb
import os
from chromadb.config import Settings
client = chromadb.Client(Settings(chroma_db_impl="duckdb+parquet",
persist_directory=os.path.join(
os.path.dirname(__file__), "db")
))
def get_client():
return client
def search(query: str) -> str:
return "d"

548
scripts/build-schemas.ts Normal file
View File

@@ -0,0 +1,548 @@
/**
* @file 用来生成 json-schemas
*/
import fs from 'fs';
import path from 'path';
import {
ObjectTypeFormatter,
LiteralUnionTypeFormatter,
IntersectionTypeFormatter,
IntersectionType,
SubTypeFormatter,
TypeFormatter,
UnionType,
derefType,
BaseType,
ObjectType,
Definition,
createFormatter,
SchemaGenerator,
createProgram,
createParser,
StringType,
LiteralType,
Context,
Schema,
uniqueArray,
ReferenceType,
ChainNodeParser,
IntersectionNodeParser
} from 'ts-json-schema-generator';
import ts from 'typescript';
import mkdirp from 'mkdirp';
import isPlainObject from 'lodash/isPlainObject';
/**
* 程序主入口
*/
async function main() {
const dir = path.join(__dirname, '../packages/amis/src');
const outDir = path.join(__dirname, '../packages/amis/');
const tsConfig = path.join(
__dirname,
'../packages/amis/tsconfig-for-declaration.json'
);
const config = {
path: path.join(dir, 'Schema.ts'),
tsconfig: tsConfig,
type: 'RootSchema',
skipTypeCheck: true,
jsDoc: 'extended' as 'extended',
additionalProperties: false
};
for (let {config: c, target, optimizeForMonaco} of [
{
config,
target: 'schema.json',
optimizeForMonaco: true
},
{
config: {
...config,
additionalProperties: true
},
target: 'schema-minimal.json'
}
]) {
// 创建 formatter 用于优化体积
const formatter = createFormatter(
c as any,
(fmt, circularReferenceTypeFormatter) => {
fmt.addTypeFormatter(
new MyNodeTypeFormatter(circularReferenceTypeFormatter)
);
fmt.addTypeFormatter(
new MyIntersectionTypeFormatter(circularReferenceTypeFormatter)
);
fmt.addTypeFormatter(new MyLiteralUnionTypeFormatter());
}
);
const program = createProgram(c as any);
const parser = createParser(program, c as any, prs => {
prs.addNodeParser(
new MyIntersectionNodeParser(
program.getTypeChecker(),
prs as ChainNodeParser
)
);
});
const generator = new SchemaGenerator(program, parser, formatter, c);
let schema = generator.createSchema(c.type);
if (optimizeForMonaco) {
convertAnyOfItemToConditionalItem(schema, [
'ActionSchema',
'CRUDSchema',
'CRUD2Schema',
'SchemaObject'
]);
schema = optimizeConditions(schema);
}
const outputFile = path.join(outDir, target);
mkdirp(path.dirname(outputFile));
fs.writeFileSync(outputFile, JSON.stringify(schema, null, 2));
}
}
class MyIntersectionNodeParser extends IntersectionNodeParser {
public createType(node: any, context: Context) {
const types = (node as ts.IntersectionType).types.map(subnode =>
this.childNodeParser.createType(subnode as any, context)
);
if (types.some(t => t.getName() === 'ActionSchema')) {
return new IntersectionType(types);
}
return super.createType(node, context);
}
}
class MyNodeTypeFormatter extends ObjectTypeFormatter {
getDefinition(type: ObjectType): Definition {
const types = type.getBaseTypes();
if (types.length === 0) {
return this.getObjectDefinition(type);
}
// 体积太大了,只能这样
const definition: Definition = {
allOf: [
this.getObjectDefinition(type),
...types.map((t: BaseType) => this.childTypeFormatter.getDefinition(t))
]
};
const firstType: Definition = definition.allOf![0] as any;
if (
firstType.type === 'object' &&
firstType.additionalProperties === false
) {
definition.allOf = definition.allOf?.map(item => {
if (typeof item === 'object' && item != null) {
if (item.$ref) {
item.additionalProperties = true;
} else {
delete item.additionalProperties;
}
}
return item;
});
definition.additionalProperties = false;
definition.properties = types.reduce((result, baseType) => {
const other = this.childTypeFormatter.getDefinition(
derefType(baseType)!
);
return {
...(other.properties
? this.getObjectBriefDefinition(other.properties)
: {}),
...result
};
}, this.getObjectBriefDefinition(firstType.properties));
if (firstType.required) {
definition.required = firstType.required;
}
if (
firstType.properties &&
definition.required &&
Object.keys(firstType.properties!).join(',') ===
definition.required.join(',')
) {
definition.allOf!.shift();
}
}
return definition;
}
getObjectBriefDefinition(properties: any): Record<string, any> {
return Object.keys(properties || {}).reduce((result, key) => {
const item = properties[key];
if (isIdentityField(item, key, properties)) {
result[key] = {
...item
};
delete result[key].description;
} else if (!result[key]) {
result[key] = true;
}
return result;
}, {} as Record<string, any>);
}
}
class MyLiteralUnionTypeFormatter extends LiteralUnionTypeFormatter {
getDefinition(type: UnionType): Definition {
const types = type.getTypes();
let definition = super.getDefinition(type);
// 表单项的 label 格式转换不对,针对这种场景修复
// label: string | false;
// 如以上定义会被处理成
// {type: "boolean", enum: [false]}
if (
!definition.anyOf &&
types.some(
item =>
item instanceof StringType && item.getPreserveLiterals() === false
) &&
types.some(item => item instanceof LiteralType && !item.isString())
) {
definition = {
anyOf: [{type: 'string'}, definition]
};
}
return definition;
}
}
class MyIntersectionTypeFormatter extends IntersectionTypeFormatter {
getDefinition(type: IntersectionType): Definition {
const types = type.getTypes();
if (
types.some(item =>
['SchemaObject', 'ActionSchema'].includes(item.getName())
)
) {
return {
allOf: types.map(item => {
const subType = this.childTypeFormatter.getDefinition(item);
return {
...subType,
additionalProperties: true
};
})
};
}
return super.getDefinition(type);
}
}
function isIdentityField(item: any, key: string, host: any) {
if (item?.const) {
return true;
} else if (item?.enum && /^(?:type|mode|actionType)$/i.test(key)) {
return !!(
key !== 'mode' ||
(host.type && ['crud', 'crud2'].includes(host.type.const))
);
}
return false;
}
function convertAnyOfItemToConditionalItem(schema: any, list: Array<string>) {
list.forEach(key => {
if (!Array.isArray(schema.definitions[key]?.anyOf)) {
return;
}
const list: Array<any> = schema.definitions[key].anyOf.concat();
const allOf: Array<any> = [];
while (list.length) {
const item: any = list.shift()!;
if (item.$ref) {
const src = schema.definitions[item.$ref.replace('#/definitions/', '')];
if (!src) {
console.log(`${item.$ref} not found`);
return;
}
if (Array.isArray(src.anyOf)) {
list.unshift.apply(list, src.anyOf);
continue;
}
const constProps = pickConstProperties(src);
Object.keys(constProps).length &&
allOf.push({
if: {
properties: constProps,
required: Object.keys(constProps)
},
then: item
});
} else {
const constProps = pickConstProperties(item);
Object.keys(constProps).length &&
allOf.push({
if: {
properties: constProps,
required: Object.keys(constProps)
},
then: item
});
}
}
schema.definitions[key] = {
allOf
};
});
}
function pickConstProperties(schema: any): any {
if (!schema.properties && schema.allOf?.[0]?.properties) {
return pickConstProperties(schema.allOf[0]);
}
const keys = Object.keys(schema.properties || {});
const filteredProperties: any = {};
keys.forEach((key: string) => {
if (isIdentityField(schema.properties[key], key, schema.properties)) {
filteredProperties[key] = {
...schema.properties[key]
};
delete filteredProperties[key].description;
}
});
return filteredProperties;
}
// 优化条件逻辑,主要处理 action 与 schemaObject 部分的 if
// 避免 monaco-editor 无法精确定位
function optimizeConditions(schema: any) {
const actionSchema = schema.definitions.ActionSchema;
const actionSchemaAllOf = actionSchema.allOf;
delete actionSchema.allOf;
actionSchema.if = {
required: ['actionType']
};
actionSchema.then = {
allOf: actionSchemaAllOf.map((item: any) => {
delete item.if.properties.type;
item.if.required = item.if.required.filter(
(item: string) => item !== 'type'
);
return item;
})
};
actionSchema.else = {
$ref: '#/definitions/VanillaAction'
};
const CRUDSchema = schema.definitions.CRUDSchema;
const CRUDSchemaAllOf = CRUDSchema.allOf;
delete CRUDSchema.allOf;
CRUDSchema.if = {
required: ['mode']
};
CRUDSchema.then = {
allOf: CRUDSchemaAllOf.map((item: any) => {
delete item.if.properties.type;
item.if.required = item.if.required.filter(
(item: string) => item !== 'type'
);
return item;
})
};
CRUDSchema.else = {
$ref: '#/definitions/CRUDTableSchema'
};
const schemaObjectSchema = schema.definitions.SchemaObject;
schemaObjectSchema.allOf = schemaObjectSchema.allOf
.filter(
(item: any) => !(item.if.properties.type && item.if.properties.actionType)
)
.map((item: any) => {
Object.keys(item.if.properties || {}).forEach(key => {
delete item.if.properties[key].description;
});
return item;
});
schemaObjectSchema.allOf.push({
if: {
properties: {
type: {enum: ['action', 'button', 'submit', 'reset']}
},
required: ['type']
},
then: {
$ref: '#/definitions/ActionSchema'
}
});
schemaObjectSchema.allOf.push({
if: {
properties: {
type: {const: 'crud'}
},
required: ['type']
},
then: {
$ref: '#/definitions/CRUDSchema'
}
});
schema.definitions.FormSchema.allOf = schema.definitions.BaseFormSchema.allOf;
schema.definitions.VanillaAction.allOf[1].additionalProperties = true;
// delete schema.definitions.FormSchemaBase.additionalProperties;
// delete schema.definitions.BaseSchemaWithoutType.additionalProperties;
// delete schema.definitions.FormSchema.additionalProperties;
// 替换 SchemaObject 与 ActionSchema 的定义
schema.definitions.SchemaObjectLoose = {
allOf: schema.definitions.SchemaObject.allOf.map((item: any) => {
return {
...item,
then: {
...item.then,
additionalProperties: true
}
};
})
};
schema.definitions.ActionSchemaLoose = {
if: schema.definitions.ActionSchema.if,
then: {
allOf: schema.definitions.ActionSchema.then.allOf.map((item: any) => {
return {
...item,
then: {
...item.then,
additionalProperties: true
}
};
})
},
else: schema.definitions.ActionSchema.else
};
schema = JSONValueMap(schema, item => {
if (
item?.$ref === '#/definitions/SchemaObject' &&
item.additionalProperties === true
) {
return {
$ref: '#/definitions/SchemaObjectLoose'
};
} else if (
item?.$ref === '#/definitions/ActionSchema' &&
item.additionalProperties === true
) {
return {
$ref: '#/definitions/ActionSchemaLoose'
};
}
return item;
});
return schema;
}
/**
* 每层都会执行,返回新的对象,新对象不会递归下去
* @param json
* @param mapper
* @returns
*/
export function JSONValueMap(
json: any,
mapper: (
value: any,
key: string | number,
host: Object,
stack: Array<Object>
) => any,
deepFirst: boolean = false,
stack: Array<Object> = []
) {
if (!isPlainObject(json) && !Array.isArray(json)) {
return json;
}
const iterator = (
origin: any,
key: number | string,
host: any,
stack: Array<any> = []
) => {
if (deepFirst) {
const value = JSONValueMap(origin, mapper, deepFirst, stack);
return mapper(value, key, host, stack) ?? value;
}
let mapped: any = mapper(origin, key, host, stack) ?? origin;
// 如果不是深度优先,上层的对象都修改了,就不继续递归进到新返回的对象了
if (mapped === origin) {
return JSONValueMap(origin, mapper, deepFirst, stack);
}
return mapped;
};
if (Array.isArray(json)) {
let modified = false;
let arr = json.map((value, index) => {
let newValue: any = iterator(value, index, json, [json].concat(stack));
modified = modified || newValue !== value;
return newValue;
});
return modified ? arr : json;
}
let modified = false;
const toUpdate: any = {};
Object.keys(json).forEach(key => {
const value: any = json[key];
let result: any = iterator(value, key, json, [json].concat(stack));
if (result !== value) {
modified = true;
toUpdate[key] = result;
}
});
return modified
? {
...json,
...toUpdate
}
: json;
}
main().catch(e => {
console.error(e);
});

11
scripts/code-md-parser.js Normal file
View File

@@ -0,0 +1,11 @@
const parserMarkdown = require('./md-parser');
module.exports = function (content, file) {
const markdowns = [];
content.replace(/\/\*\!markdown\n([\s\S]+?)\*\//g, function (_, md) {
markdowns.push(md.trim());
});
return parserMarkdown(markdowns.join('\n\n'), file);
};

26
scripts/convert-citydb.ts Normal file
View File

@@ -0,0 +1,26 @@
/**
* 将 https://lbs.amap.com/api/webservice/download 中的 json 转成 json 格式
*/
import {raw} from 'express';
import process from 'node:process';
import XLSX from 'xlsx';
const file = process.argv[2];
const workbook = XLSX.readFile(file);
const worksheet = workbook.Sheets[workbook.SheetNames[0]];
const rawData = XLSX.utils.sheet_to_json(worksheet) as Array<{
中文名: string;
adcode: string;
}>;
const cityDb: Record<string, string> = {};
for (const row of rawData) {
cityDb[row.adcode] = row['中文名'];
}
console.log(JSON.stringify(cityDb, null, 2));

214
scripts/embed-packager.js Normal file
View File

@@ -0,0 +1,214 @@
/* eslint-disable */
var rLinkScript =
/(<!(?:--)?\[[\s\S]*?<\!\[endif\](?:--)?>|<!--[\s\S]*?(?:-->|$))|(?:(\s*<script([^>]*)>([\s\S]*?)<\/script>)|(?:\s*(<link([^>]*?)(?:\/)?>)|(<style([^>]*)>([\s\S]*?)<\/style>)))(<!--ignore-->)?\n?/gi;
var rScriptType = /type=('|")(.*?)\1/i;
var rSrcHref = /\s*(?:src|href)=('|")(.+?)\1/i;
var rRefStyle = /rel=('|")stylesheet\1/i;
var path = require('path');
var css = require('css');
var rSourceMap =
/(?:\/\/\#\s*sourceMappingURL[^\r\n\'\"]*|\/\*\#\s*sourceMappingURL[^\r\n\'\"]*\*\/)(?:\r?\n|$)/gi;
var caches = {};
var createResource = fis.require('postpackager-loader/lib/resource.js');
const package = require('../packages/amis/package.json');
function prefixCss(code, prefix) {
var cssAst = css.parse(code);
prefixNode(cssAst);
return css.stringify(cssAst, {
compress: true
});
function prefixSelector(sel) {
if (sel.match(/^@/)) return sel;
if (sel.match(/^:root/)) return sel;
var m = sel.match(/(^| )(body|html)($|\W.*)/i);
if (m) return m[1] + prefix + m[3];
else if (sel.match(/^\.is\-modalOpened/))
return sel.replace(
/^\.is\-modalOpened\s/,
'.is-modalOpened ' + prefix + ' '
);
else if (
sel.match(
/^(?:\.fr-|\.fa|\.tox|\.AMISDebug|\.monaco-|\.vs-dark|\.hc-black|\.vs\b|\.cursor-|::|\.context-view|\.menubar|\.fullscreen|\.colorpicker-)/
)
)
return sel;
else return prefix + ' ' + sel;
}
function prefixNode(node) {
if (node.selectors) {
node.selectors = node.selectors.map(prefixSelector);
} else if (node.stylesheet) {
node.stylesheet.rules.forEach(prefixNode);
} else if (node.rules) {
node.rules.forEach(prefixNode);
}
}
}
function unicodeJs(str) {
return str.replace(
/([\u4E00-\u9FA5]|[\uFE30-\uFFA0]|[\u2019])/g,
function (_, value) {
return '\\u' + value.charCodeAt(0).toString(16);
}
);
}
module.exports = function (ret, pack, settings, opt) {
var root = fis.project.getProjectPath();
var tpl = ret.pkg['/examples/sdk-placeholder.html'];
tpl.skiped = true;
if (tpl && tpl._fromCache && caches[tpl.id]) {
tpl.setContent(caches[tpl.id]);
return;
} else if (!tpl) {
return;
}
var mapping = {};
var contents = tpl.getContent();
var cssContents = [];
var jsContents = '';
var entryJs = '';
var resource = tpl._resource;
var files = ret.pkg;
Object.keys(files).forEach(function (subpath) {
var file = files[subpath];
mapping[file.getUrl()] = file;
});
contents.replace(
rLinkScript,
function (
all,
comment,
script,
attrs,
body,
link,
lattrs,
style,
sattrs,
sbody,
ignored
) {
// 忽略注释。
if (comment || ignored) {
return all;
}
if (script && !body.trim() && rSrcHref.test(attrs)) {
all = '';
let src = RegExp.$2;
let file = resource.getFileByUrl(src);
if (!file) {
file = resource.getFileByUrl(
fis.util(path.join(path.dirname(tpl.release), src))
);
}
if (!file) {
file = mapping[src];
}
if (file) {
file.skiped = true;
let contents = file.getContent();
if (/_map\.js$/.test(file.subpath)) {
contents = `(function() {
${contents.replace(
/\"url\"\s*\:\s*('|")(\.\/.*?)\1/g,
function (_, quote, value) {
return `"url": amis['sdk@${
package.version
}BasePath'] + ${quote}${value.substring(1)}${quote}`;
}
)}
})()`;
}
jsContents += contents + ';\n';
}
} else if (
(script && !rScriptType.test(attrs)) ||
(rScriptType.test(attrs) &&
~['text/javascript', 'application/javascript'].indexOf(
RegExp.$2.toLowerCase()
))
) {
entryJs += ';' + body;
all = '';
} else if (link && rRefStyle.test(lattrs) && rSrcHref.test(lattrs)) {
var href = RegExp.$2;
let file = resource.getFileByUrl(href);
if (!file) {
file = resource.getFileByUrl(
fis.util(path.join(path.dirname(tpl.release), href))
);
}
if (!file) {
file = mapping[href];
}
if (file) {
cssContents.push({
name: file.basename,
content: file.getContent()
});
file.skiped = true;
}
all = '';
} else if (style && sbody.trim()) {
cssContents.push({
name: 'inline',
content: sbody
});
all = '';
}
return all;
}
);
jsContents = jsContents.replace(rSourceMap, '');
jsContents = unicodeJs(jsContents);
let jsFile = fis.file(root, 'sdk.js');
jsFile.setContent(jsContents);
ret.pkg[jsFile.subpath] = jsFile;
// cssContents = prefixCss(cssContents, '.amis-scope');
// let cssFile = fis.file(root, 'sdk.css');
// cssFile.setContent(cssContents);
// ret.pkg[cssFile.subpath] = cssFile;
const themes = ['ang', 'cxd', 'dark', 'antd'];
themes.forEach(function (theme) {
const rest = themes.filter(a => a !== theme).map(item => item + '.scss');
let contents = cssContents
.filter(item => !rest.includes(item.name))
.map(item => item.content)
.join('\n');
contents = prefixCss(contents, '.amis-scope');
let cssFile = fis.file(root, (theme === 'cxd' ? 'sdk' : theme) + '.css');
cssFile.setContent(contents);
ret.pkg[cssFile.subpath] = cssFile;
});
// tpl.setContent(contents);
caches[tpl.id] = contents;
};

62
scripts/fis3plugin.ts Normal file
View File

@@ -0,0 +1,62 @@
import MagicString from 'magic-string';
import {createFilter} from 'rollup-pluginutils';
import type {Plugin} from 'vite';
export default function fis3replace(
options: {
include?: any;
exclude?: any;
sourcemap?: boolean;
sourceMap?: boolean;
} = {}
): Plugin {
const filter = createFilter(options.include, options.exclude);
return {
name: 'fis3',
enforce: 'pre',
apply: 'serve',
transform(code: string, id: string) {
if (!filter(id)) return null;
const magicString = new MagicString(code);
let hasReplacements = false;
let start;
let end;
code = code.replace(
/(__uri|__inline)\(\s*('|")(.*?)\2\s*\)/g,
(_, directive, quote, target, index) => {
hasReplacements = true;
start = index;
end = start + _.length;
if (directive === '__uri') {
let replacement = `new URL(${quote}${target}${quote}, import.meta.url).href`;
magicString.overwrite(start, end, replacement);
} else if (directive === '__inline') {
let varname = target
.replace(/[^a-zA-Z0-9]/g, '')
.replace(/^\d+/, '');
magicString.prepend(`import ${varname} from '${target}?inline';\n`);
magicString.overwrite(start, end, `${varname}`);
}
return _;
}
);
if (!hasReplacements) return null;
const result: any = {code: magicString.toString()};
if (options.sourceMap !== false && options.sourcemap !== false)
result.map = magicString.generateMap({hires: true});
return result;
}
};
}

View File

@@ -0,0 +1,42 @@
/**
* @file 用于生成 helper 文档中的 class 列表,需要在 scss 目录下运行。
* 比如 node ../scripts/generate-helper-table.js helper/border/_border-style.scss
*/
const scssFile = process.argv[2];
const fs = require('fs');
const css = require('css');
const execSync = require('child_process').execSync;
fs.writeSync(
fs.openSync('./tmp.scss', 'w'),
`
@import './functions';
@import './variables';
@import './mixins';
@import '${scssFile}';
`
);
let generateCSS = execSync(`../node_modules/.bin/sass tmp.scss`).toString();
generateCSS = generateCSS.replace(/\/\*\!markdown[\s\S]*?\*\//g, '');
let parseCSS = css.parse(generateCSS);
console.log(`| Class | Properties |
| ----------- | ------------------------ |`);
for (let rule of parseCSS.stylesheet.rules) {
if (rule.type === 'rule') {
let className = rule.selectors[0].replace('.', '');
let declarations = rule.declarations
.map(declaration => {
return `${declaration.property}: ${declaration.value}`;
})
.join('; ');
console.log(`| ${className} | ${declarations} |`);
}
}
fs.unlinkSync('./tmp.scss');

View File

@@ -0,0 +1,47 @@
/**
* 记录发布时的commit信息,用于区分内网版本包之间的差异
*/
import path from 'path';
import {writeFileSync} from 'fs';
import {execSync} from 'child_process';
let outputFileName = process.argv[2];
if (!outputFileName) {
outputFileName = 'revision.json';
}
try {
const rootDir = execSync('git rev-parse --show-toplevel').toString().trim();
// 分支
const branch = execSync('git rev-parse --abbrev-ref HEAD').toString().trim();
// commit id
const SHA1 = execSync('git rev-parse --short HEAD').toString().trim();
// commit body
const commit = execSync(
`git log -n 1 --date=format:'%Y-%m-%d %H:%M:%S' --format="%s%n%an%n%cd"`
)
.toString()
.trim();
const [message, author, date] = commit.split('\n');
const content = JSON.stringify(
{branch, SHA1, message, author, date},
undefined,
2
);
writeFileSync(path.join(rootDir, outputFileName), content, {
encoding: 'utf8'
});
console.log(
'\x1b[32m%s\x1b[0m',
'✨ [amis] revision.json generated successfully!'
);
process.exit();
} catch (error) {
console.log('\x1b[31m%s\x1b[0m', '❌ [amis] revision.json failed to write!');
process.exit(1);
}

View File

@@ -0,0 +1,16 @@
/**
* 用于生成 sdk 里的多语言,目前只有德语
*/
const fs = require('fs');
try {
const localeFile = fs.readFileSync(process.argv[2], 'utf8');
console.log(
localeFile
.replace(`import {register} from '../locale';`, '')
.replace('register(', `amisRequire('amis').registerLocale(`)
);
} catch (err) {
console.error(err);
}

View File

@@ -0,0 +1,41 @@
/**
* @file 生成给前端全文搜索用的文件
* @author wuduoyi
*/
const glob = require('glob');
const fs = require('fs');
let yaml = require('js-yaml');
var rYml = /^\s*---([\s\S]*?)---\s/;
const resultData = {docs: []};
glob('./docs/**/*.md', {}, function (er, docs) {
for (const doc of docs) {
let content = fs.readFileSync(doc, {encoding: 'utf8'});
let m = rYml.exec(content);
let info = {};
if (m && m[1]) {
info = yaml.load(m[1]);
content = content.substring(m[0].length);
}
const title = info.title || doc;
// todo: 属性列表单独处理,检索的时候优先检索
resultData.docs.push({
title: title,
// 去掉注释、换行、图片等
body: content
.replace(/<\!---.+-->/g, '')
.replace(/!?\[.*\]\(.*\)/g, '')
.replace(/```[^`]*```/g, ''),
path: doc
.slice(1)
.replace('.md', '')
.replace('/docs/zh-CN/components/', '/zh-CN/components/')
.replace('/docs/zh-CN/style/', '/zh-CN/style/')
.replace('/docs/zh-CN/', '/zh-CN/docs/')
});
}
fs.writeFileSync('./examples/docs.json', JSON.stringify(resultData));
});

36
scripts/get-all-docs.js Normal file
View File

@@ -0,0 +1,36 @@
/**
* 获取所有文档和 json 示例,生成到一个文件里,用于继续预训练
*/
const fs = require('fs');
const path = require('path');
const glob = require('glob');
const docs = path.resolve(__dirname, '..', 'docs');
const examples = path.resolve(__dirname, '..', 'examples');
const outputFile = 'amis_docs.txt';
glob(docs + '/**/*.md', {}, (err, files) => {
for (const file of files) {
const data = fs.readFileSync(file, {encoding: 'utf8', flag: 'r'});
fs.appendFileSync(outputFile, data + '\n\n');
}
});
// glob(examples + '/**/*.jsx', {}, (err, files) => {
// for (const file of files) {
// try {
// const content = fs.readFileSync(file, {encoding: 'utf8', flag: 'r'});
// if (content.indexOf('export default') !== -1) {
// const schema = Function(content.replace('export default', 'return'))();
// const json = JSON.stringify(schema, null, 2);
// console.log(file);
// fs.appendFileSync(outputFile, json + '\n\n');
// }
// } catch (e) {
// // console.log(e);
// }
// }
// });

103
scripts/getChangeLog.ts Normal file
View File

@@ -0,0 +1,103 @@
import fs from 'fs';
import path from 'path';
import util from 'util';
import {exec as nodeExec} from 'child_process';
const exec = util.promisify(nodeExec);
const pkg = '@fex/amis';
const folder = path.join(path.dirname(__filename), '../node_modules');
export async function main(from: string, to: string, useMarkdown: boolean = false) {
const toBeClearedQueue = [];
let {stdout: packFromStdout} = await exec(
`npm pack ${pkg}@${from} --pack-destination="${folder}"`
);
const fromFilname = packFromStdout.trim();
const fromFolder = `${folder}/amis_${from}`;
const fromTgz = path.join(folder, fromFilname);
await exec(`mkdir -p ${fromFolder}`);
await exec(`tar -xzvf ${fromTgz} -C ${fromFolder}`);
toBeClearedQueue.push(fromFolder, fromTgz);
const fromRevision = JSON.parse(
fs.readFileSync(fromFolder + '/package/revision.json', 'utf-8').trim()
);
const {stdout: packToStdout} = await exec(
`npm pack ${pkg}@${to} --pack-destination="${folder}"`
);
const toFilname = packToStdout.trim();
const toFolder = `${folder}/amis_${to}`;
const toTgz = path.join(folder, toFilname);
await exec(`mkdir -p ${toFolder}`);
await exec(`tar -xzvf ${toTgz} -C ${toFolder}`);
toBeClearedQueue.push(toFolder, toTgz);
const toRevision = JSON.parse(
fs.readFileSync(toFolder + '/package/revision.json', 'utf-8').trim()
);
const {stdout: logs} = await exec(
`git log ${fromRevision.SHA1}...${toRevision.SHA1}^ --oneline`
);
const lines = logs.split('\n');
const commits = lines
.map(line => {
const [SHA1, ...message] = line.split(' ');
const msg = message.join(' ');
if (msg.startsWith('Merge') || msg.startsWith('bump:')) {
return '';
}
return msg;
})
.filter(item => item);
const prefixes = ['feat', 'feats', 'feat(amis)', 'feat(amis-editor)', 'fix', 'fix(amis)', 'fix(amis-editor)', 'chore', 'chore(amis)', 'chore(amis-editor)', 'styles', 'style', 'style(amis)', 'style(amis-editor)', 'docs', 'doc', 'docs(amis)', 'docs(amis-editor)', 'perf', 'refactor', 'revert',];
/** 按照commit类型排序 */
commits.sort((lhs, rhs) => {
let lhsPrefixIndex = prefixes.findIndex(p => p === lhs.split(/\s*[::]+\s*/)[0]);
let rhsPrefixIndex = prefixes.findIndex(p => p === rhs.split(/\s*[::]+\s*/)[0]);
/** 找不到对应的标签就排队尾 */
if (lhsPrefixIndex === -1) {
lhsPrefixIndex = prefixes.length + 1;
}
if (rhsPrefixIndex === -1) {
rhsPrefixIndex = prefixes.length + 1;
}
return lhsPrefixIndex - rhsPrefixIndex;
})
console.log(
'\x1b[32m%s\x1b[0m',
`✨ [@fex/amis] ${from} ===> ${to} change log:`
);
if (useMarkdown) {
console.log(commits.map(i => `* ${i}`).join('\n'));
} else {
console.log(commits.join('\n'));
}
/** 清除缓存的文件和文件夹 */
await exec(`npx rimraf ${toBeClearedQueue.join(' ')}`);
console.log(
'\x1b[32m%s\x1b[0m',
'✨ [@fex/amis] Temporarily cached files have been successfully cleared!'
);
}
main.apply(null, process.argv.slice(2)).catch((err: any) => {
console.error(err);
process.exit(1);
});

10
scripts/imagemin.sh Normal file
View File

@@ -0,0 +1,10 @@
#!/bin/bash
for f in $(find . -type d \( -path ./node_modules -o -path ./dist -o -path ./esm -o -path ./lib -o -path ./packages/amis/sdk \) -prune -type f -o -name "*.svg"); do
#sed -i '' -e "s/@fex\///g" $f
echo "imagemin $f"
echo $(imagemin $f) >$f
# git checkout $f
# imagemin $f > $f
done

253
scripts/markdownPlugin.ts Normal file
View File

@@ -0,0 +1,253 @@
import type {Plugin} from 'vite';
import * as marked from 'marked';
import * as prism from 'prismjs';
import * as yaml from 'js-yaml';
let loadLanguages = require('prismjs/components/');
loadLanguages([
'bash',
'javascript',
'java',
'python',
'jsx',
'tsx',
'css',
'markup',
'json'
]);
const rYml = /^\s*---([\s\S]*?)---\s/;
const renderer = new marked.Renderer();
(marked as any).setOptions({
renderer: renderer,
gfm: true,
tables: true,
breaks: false,
pedantic: false,
smartLists: true,
smartypants: false
});
// Synchronous highlighting with prism.js
(marked as any).setOptions({
highlight: function (code: string, lang: string) {
if (lang) {
return prism.highlight(code, prism.languages[lang], lang);
} else {
return code;
}
}
});
// renderer.table = function(header, body) {
// return '<table class="table table-striped">\n'
// + '<thead>\n'
// + header
// + '</thead>\n'
// + '<tbody>\n'
// + body
// + '</tbody>\n'
// + '</table>\n';
// };
renderer.link = function (href: string, title: string, text: string) {
if ((this as any).options.sanitize) {
try {
var prot = decodeURIComponent(unescape(href))
.replace(/[^\w:]/g, '')
.toLowerCase();
} catch (e) {
return '';
}
if (prot.indexOf('javascript:') === 0 || prot.indexOf('vbscript:') === 0) {
return '';
}
}
if (href && href[0] === '#') {
href =
'#' +
encodeURIComponent(
href
.substring(1)
.toLowerCase()
.replace(/[^\u4e00-\u9fa5_a-zA-Z0-9]+/g, '-')
);
}
var out = '<a href="' + href + '"';
if (title) {
out += ' title="' + title + '"';
}
out += '>' + text + '</a>';
return out;
};
function markdown2js(content: string, file: string) {
var raw = content;
var m = rYml.exec(content);
var info: any = {};
if (m && m[1]) {
info = (yaml as any).load(m[1]);
content = content.substring(m[0].length);
}
var toc: any = {
label: '目录',
type: 'toc',
children: [],
level: 0
};
var stack = [toc];
renderer.heading = function (text, level) {
var escapedText = encodeURIComponent(
text.toLowerCase().replace(/[^\u4e00-\u9fa5_a-zA-Z0-9]+/g, '-')
);
if (level < 5) {
var menu = {
label: text,
fragment: escapedText,
fullPath: '#' + escapedText,
level: level
};
while (stack.length && stack[0].level >= level) {
stack.shift();
}
stack[0].children = stack[0].children || [];
stack[0].children.push(menu);
stack.unshift(menu);
}
var anchor =
'<a class="anchor" name="' +
escapedText +
'" href="#' +
escapedText +
'" aria-hidden="true"><svg aria-hidden="true" class="octicon octicon-link" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>';
return '<h' + level + '>' + anchor + text + '</h' + level + '>';
// return '<h' + level + '><a name="' +
// escapedText +
// '" class="anchor" href="#' +
// escapedText +
// '"><span class="header-link"></span></a>' +
// text + '</h' + level + '>';
};
const placeholder: any = {};
let index = 1;
content = content
.replace(/\!\!\!include\s*\(([^\)]+?)\)\!\!\!/g, (_, val) => {
// todo
// const result = null;
// if (result) {
// // 暂时不支持嵌套 include
// return result.file.getContent();
// }
return _;
})
.replace(
/```(schema|html)(?::(.*?))?[\n|\r\n]([\s\S]*?)```/g,
function (_, lang, attr, code) {
const setting: any = {};
attr &&
attr.split(/\s+/).forEach(function (item: string) {
var parts = item.split('=');
if (parts[1] && /^('|").*\1/.test(parts[1])) {
parts[1] = parts[1].substring(1, parts[1].length - 1);
}
setting[parts[0]] = parts[1] ? decodeURIComponent(parts[1]) : '';
if (parts[0] === 'height') {
setting.height = parseInt(setting.height, 10) /*编辑器的高度*/;
attr = attr.replace(item, `height="${setting.height}"`);
}
});
// placeholder[index] = `<iframe class="doc-iframe" width="100%" height="${setting.height || 200}px" frameBorder="0" src="/play?code=${encodeURIComponent(code)}&scope=${encodeURIComponent(setting.scope)}"></iframe>`;
if (lang === 'html') {
if (~code.indexOf('<html') || ~code.indexOf('<link')) {
return _;
}
placeholder[
index
] = `<!--amis-preview-start--><div class="amis-doc"><div class="preview">${code}</div><pre><code class="lang-html">${prism.highlight(
code
.replace(/"data:(\w+\/\w+);.*?"/g, '"data:$1; ..."')
.replace(/<svg([^>]*)>[\s\S]*?<\/svg>/g, '<svg$1>...</svg>')
.replace(/class="([^"]*?)\.\.\.([^"]*?)"/g, 'class="$1..."'),
prism.languages[lang],
lang
)}</code></pre></div><!--amis-preview-end-->`;
} else {
placeholder[
index
] = `<!--amis-preview-start--><div class="amis-preview" style="min-height: ${setting.height}px"><script type="text/schema" ${attr}>${code}</script></div><!--amis-preview-end-->`;
}
return `[[${index++}]]`;
}
);
content = (marked as any)
.parse(content)
.replace(/<p>\[\[(\d+)\]\]<\/p>/g, function (_: string, id: string) {
return placeholder[id] || '';
});
// content = global.fis ? fis.compile.partial(content, file, 'html') : content;
// + `\n\n<div class="m-t-lg b-l b-info b-3x wrapper bg-light dk">文档内容有误?欢迎大家一起来编写,文档地址:<i class="fa fa-github"></i><a href="https://github.com/baidu/amis/tree/master${file.subpath}">${file.subpath}</a>。</div>`;
info.html =
'<div class="markdown-body">' +
content.replace(
/<\!\-\-amis\-preview\-(start|end)\-\-\>/g,
function (_, type) {
return type === 'start' ? '</div>' : '<div class="markdown-body">';
}
) +
'</div>';
info.raw = raw;
info.toc = toc;
return 'export default ' + JSON.stringify(info, null, 2) + ';';
}
export default function markdownPlugin(options: {} = {}): Plugin {
return {
name: 'markdown-as-js',
enforce: 'pre',
apply: 'serve',
transform(code: string, id: string) {
// if (id.endsWith('.scss') && /\/_[^\/]+\.scss$/.test(id)) {
// const markdowns: Array<string> = [];
// code.replace(
// /\/\*\!markdown\n([\s\S]+?)\*\//g,
// (_: string, md: string) => {
// markdowns.push(md.trim());
// return _;
// }
// );
// if (markdowns.length) {
// return {code: markdown2js(markdowns.join('\n'), id) as string};
// }
// return null;
// }
if (!id.endsWith('.md')) return null;
return {code: markdown2js(code, id) as string};
}
};
}

220
scripts/md-parser.js Normal file
View File

@@ -0,0 +1,220 @@
/* eslint-disable */
var marked = require('marked');
let prism = require('prismjs');
let loadLanguages = require('prismjs/components/');
loadLanguages([
'bash',
'javascript',
'java',
'python',
'jsx',
'tsx',
'css',
'markup',
'json'
]);
var yaml = (yaml = require('js-yaml'));
var rYml = /^\s*---([\s\S]*?)---\s/;
var renderer = new marked.Renderer();
marked.setOptions({
renderer: renderer,
gfm: true,
tables: true,
breaks: false,
pedantic: false,
smartLists: true,
smartypants: false
});
// Synchronous highlighting with prism.js
marked.setOptions({
highlight: function (code, lang) {
if (lang) {
return prism.highlight(code, prism.languages[lang], lang);
} else {
return code;
}
}
});
// renderer.table = function(header, body) {
// return '<table class="table table-striped">\n'
// + '<thead>\n'
// + header
// + '</thead>\n'
// + '<tbody>\n'
// + body
// + '</tbody>\n'
// + '</table>\n';
// };
renderer.link = function (href, title, text) {
if (this.options.sanitize) {
try {
var prot = decodeURIComponent(unescape(href))
.replace(/[^\w:]/g, '')
.toLowerCase();
} catch (e) {
return '';
}
if (prot.indexOf('javascript:') === 0 || prot.indexOf('vbscript:') === 0) {
return '';
}
}
if (href && href[0] === '#') {
href =
'#' +
encodeURIComponent(
href
.substring(1)
.toLowerCase()
.replace(/[^\u4e00-\u9fa5_a-zA-Z0-9]+/g, '-')
);
}
var out = '<a href="' + href + '"';
if (title) {
out += ' title="' + title + '"';
}
out += '>' + text + '</a>';
return out;
};
module.exports = function (content, file) {
var m = rYml.exec(content);
var info = {};
if (m && m[1]) {
info = yaml.load(m[1]);
content = content.substring(m[0].length);
}
var toc = {
label: '目录',
type: 'toc',
children: [],
level: 0
};
var stack = [toc];
renderer.heading = function (text, level) {
var escapedText = encodeURIComponent(
text.toLowerCase().replace(/[^\u4e00-\u9fa5_a-zA-Z0-9]+/g, '-')
);
if (level < 5) {
var menu = {
label: text,
fragment: escapedText,
fullPath: '#' + escapedText,
level: level
};
while (stack.length && stack[0].level >= level) {
stack.shift();
}
stack[0].children = stack[0].children || [];
stack[0].children.push(menu);
stack.unshift(menu);
}
var anchor =
'<a class="anchor" name="' +
escapedText +
'" href="#' +
escapedText +
'" aria-hidden="true"><svg aria-hidden="true" class="octicon octicon-link" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>';
return '<h' + level + '>' + anchor + text + '</h' + level + '>';
// return '<h' + level + '><a name="' +
// escapedText +
// '" class="anchor" href="#' +
// escapedText +
// '"><span class="header-link"></span></a>' +
// text + '</h' + level + '>';
};
const placeholder = {};
let index = 1;
content = content
.replace(/\!\!\!include\s*\(([^\)]+?)\)\!\!\!/g, (_, val) => {
const result = fis.project.lookup(val, file);
if (result) {
// 暂时不支持嵌套 include
return result.file.getContent();
}
return _;
})
.replace(
/```(schema|html)(?::(.*?))?\n([\s\S]*?)```/g,
function (_, lang, attr, code) {
const setting = {};
attr &&
attr.split(/\s+/).forEach(function (item) {
var parts = item.split('=');
if (parts[1] && /^('|").*\1/.test(parts[1])) {
parts[1] = parts[1].substring(1, parts[1].length - 1);
}
setting[parts[0]] = parts[1] ? decodeURIComponent(parts[1]) : '';
if (parts[0] === 'height') {
setting.height = parseInt(setting.height, 10) /*编辑器的高度*/;
attr = attr.replace(item, `height="${setting.height}"`);
}
});
// placeholder[index] = `<iframe class="doc-iframe" width="100%" height="${setting.height || 200}px" frameBorder="0" src="/play?code=${encodeURIComponent(code)}&scope=${encodeURIComponent(setting.scope)}"></iframe>`;
if (lang === 'html') {
if (~code.indexOf('<html') || ~code.indexOf('<link')) {
return _;
}
placeholder[
index
] = `<!--amis-preview-start--><div class="amis-doc"><div class="preview">${code}</div><pre><code class="lang-html">${prism.highlight(
code
.replace(/"data:(\w+\/\w+);.*?"/g, '"data:$1; ..."')
.replace(/<svg([^>]*)>[\s\S]*?<\/svg>/g, '<svg$1>...</svg>')
.replace(/class="([^"]*?)\.\.\.([^"]*?)"/g, 'class="$1..."'),
prism.languages[lang],
lang
)}</code></pre></div><!--amis-preview-end-->`;
} else {
placeholder[
index
] = `<!--amis-preview-start--><div class="amis-preview" style="min-height: ${setting.height}px"><script type="text/schema" ${attr}>${code}</script></div><!--amis-preview-end-->`;
}
return `[[${index++}]]`;
}
);
content = marked
.parse(content)
.replace(/<p>\[\[(\d+)\]\]<\/p>/g, function (_, id) {
return placeholder[id] || '';
});
content = fis.compile.partial(content, file, 'html');
// + `\n\n<div class="m-t-lg b-l b-info b-3x wrapper bg-light dk">文档内容有误?欢迎大家一起来编写,文档地址:<i class="fa fa-github"></i><a href="https://github.com/baidu/amis/tree/master${file.subpath}">${file.subpath}</a>。</div>`;
info.html =
'<div class="markdown-body">' +
content.replace(
/<\!\-\-amis\-preview\-(start|end)\-\-\>/g,
function (_, type) {
return type === 'start' ? '</div>' : '<div class="markdown-body">';
}
) +
'</div>';
info.toc = toc;
return 'module.exports = ' + JSON.stringify(info, null, 2) + ';';
};

93
scripts/mockApiPlugin.ts Normal file
View File

@@ -0,0 +1,93 @@
import type {Plugin} from 'vite';
// @ts-ignore
import handler from '../mock/index';
import express, {query} from 'express';
import setPrototypeOf from 'setprototypeof';
import path from 'path';
import fs from 'fs';
const app = express();
function initExpress(req: any, res: any, next: any, callback: () => void) {
if (app.enabled('x-powered-by')) res.setHeader('X-Powered-By', 'Express');
(req as any).res = res;
res.req = req;
(req as any).next = next;
setPrototypeOf(req, app.request);
setPrototypeOf(res, app.response);
res.locals = res.locals || Object.create(null);
query({})(req as any, res, callback);
}
export default function mockApiPlugin(options: {} = {}): Plugin {
return {
name: 'mock-api',
enforce: 'pre' as 'pre',
apply: 'serve' as 'serve',
configureServer(server) {
// examples/index 页面,不处理会直接吐出 html 文件
server.middlewares.use('/examples/index', async (req, res, next) => {
if (req.originalUrl !== '/examples/index') {
next();
return;
}
let template = fs.readFileSync(
path.resolve(__dirname, '../index.html'),
'utf-8'
);
template = await server.transformIndexHtml('/index.html', template);
res.statusCode = 200;
res.end(template);
});
server.middlewares.use('/api', (req, res, next) => {
initExpress(req, res, next, () => {
handler(req, res);
});
});
// 处理 app 模式
server.middlewares.use('/examples/app/', async (req, res, next) => {
if (req.originalUrl !== '/examples/app/') {
next();
return;
}
let template = fs.readFileSync(
path.resolve(__dirname, '../examples/app/index-vite.html'),
'utf-8'
);
template = await server.transformIndexHtml(
'/examples/app/index-vite.html',
template
);
res.statusCode = 200;
res.end(template);
});
server.middlewares.use('/schema.json', (req, res, next) => {
initExpress(req, res, next, () => {
const filepath = path.resolve(
__dirname,
'../packages/amis/schema.json'
);
if (!fs.existsSync(filepath)) {
(res as any).json({
$schema: 'http://json-schema.org/draft-07/schema#',
type: 'object',
description:
'amis/schema.json 还没有构建,请执行 `npm run build-schemas --workspace amis` 后看效果'
});
return;
}
(res as any).sendFile(filepath);
});
});
}
};
}

8
scripts/scss-ie11.js Normal file
View File

@@ -0,0 +1,8 @@
const postcss = require('postcss');
const postcssCustomProperties = require('postcss-custom-properties');
module.exports = function (content, file) {
// console.log(content);
return postcss([postcssCustomProperties({preserve: false})]).process(content)
.css;
};

71
scripts/sdk-size.js Normal file
View File

@@ -0,0 +1,71 @@
/**
* 用于简单计算 sdk 各个模块的大小
*/
const readline = require('readline');
const fs = require('fs');
const readInterface = readline.createInterface({
input: fs.createReadStream(process.argv[2]),
console: false
});
let currentModule = '';
let moduleSizeMap = {};
readInterface
.on('line', line => {
if (line.startsWith(`;/*`)) {
currentModule = line.trim();
}
if (currentModule in moduleSizeMap) {
moduleSizeMap[currentModule] += line.length;
} else {
moduleSizeMap[currentModule] = line.length;
}
})
.on('close', () => {
const moduleSize2Map = {};
let sizeArray = [];
for (let module in moduleSizeMap) {
sizeArray.push([module, moduleSizeMap[module]]);
const parts = module.substring(4, module.length - 2).split('/');
while (parts.length > 1 && parts[parts.length - 2] !== 'node_modules') {
if (
parts[parts.length - 3] === 'node_modules' &&
parts[parts.length - 2][0] === '@'
) {
break;
}
parts.pop();
}
if (parts[0] === 'node_modules') {
parts.shift();
}
const moduleName = parts.join('/');
moduleSize2Map[moduleName] = moduleSize2Map[moduleName] || 0;
moduleSize2Map[moduleName] += moduleSizeMap[module];
}
sizeArray.sort(function (a, b) {
return a[1] - b[1];
});
for (size of sizeArray) {
console.log(size[0], size[1]);
}
console.log('\n\n\npackages\n\n');
const sizeArray2 = [];
for (let module in moduleSize2Map) {
sizeArray2.push([module, moduleSize2Map[module]]);
}
sizeArray2.sort(function (a, b) {
return a[1] - b[1];
});
for (size of sizeArray2) {
console.log(size[0], size[1]);
}
});

View File

@@ -0,0 +1,90 @@
const parse = require('@babel/parser').parse;
const traverse = require('@babel/traverse').default;
const generate = require('@babel/generator').default;
const types = require('@babel/types');
module.exports = function transformNodeEnvInline(file) {
// if (file.subpath !== '/node_modules/mobx/lib/index.js') {
// return;
// }
if (!file.isJsLike || !file.isMod) {
return;
}
let contents = file.getContent();
const idx = contents.indexOf('process.env.NODE_ENV');
if (idx === -1) {
return;
}
const idx2 = contents.indexOf('require(', idx);
if (idx2 === -1) {
return;
}
const env = file.optimizer ? 'production' : 'development';
contents = contents.replace(/typeof\s+process\b/g, () =>
JSON.stringify('object')
);
const ast = parse(contents, {
sourceType: 'module'
});
traverse(ast, {
// 参考: https://www.trickster.dev/post/javascript-ast-manipulation-with-babel-removing-unreachable-code/
// 参考:https://github.com/babel/minify/blob/master/packages/babel-plugin-transform-node-env-inline/src/index.js
MemberExpression(path) {
if (path.matchesPattern('process.env.NODE_ENV')) {
path.replaceWith(types.valueToNode(env));
let parentPath = path.parentPath;
if (parentPath.isBinaryExpression()) {
const evaluated = parentPath.evaluate();
if (evaluated.confident) {
parentPath.replaceWith(types.valueToNode(evaluated.value));
}
}
let ifStatement = null;
while (parentPath) {
if (
parentPath.isIfStatement() ||
parentPath.isConditionalExpression()
) {
ifStatement = parentPath;
}
parentPath = parentPath.parentPath;
}
if (ifStatement) {
let isTruthy = ifStatement.get('test').evaluateTruthy();
const node = ifStatement.node;
if (isTruthy) {
if (types.isBlockStatement(node.consequent)) {
ifStatement.replaceWithMultiple(node.consequent.body);
} else {
ifStatement.replaceWith(node.consequent);
}
} else if (node.alternate != null) {
if (types.isBlockStatement(node.alternate)) {
ifStatement.replaceWithMultiple(node.alternate.body);
} else {
ifStatement.replaceWith(node.alternate);
}
} else {
ifStatement.remove();
}
}
}
}
});
contents = generate(ast, {}).code;
file.setContent(contents);
// console.log('\n', file.subpath, '\n', file.getContent());
// process.exit(1);
};

View File

@@ -0,0 +1,56 @@
import MagicString from 'magic-string';
import type {Plugin} from 'vite';
/**
* 因为这个 mobile.html 也作用于 fis3 编译,不能直接修改源码
* 所以靠这个插件来改成让 vite 支持
* @param options
* @returns
*/
export default function transformMobileHtml(options: {} = {}): Plugin {
return {
name: 'amis-transform-mobile-html',
enforce: 'pre',
apply: 'serve',
transformIndexHtml(html: string, file) {
if (
file.path === '/examples/mobile.html' ||
file.path === '/examples/index.html'
) {
html = html.replace(/href=('|")(.*?)\1/g, (_, quote, value) => {
if (/^amis\/lib\/themes\/(.*)\.css$/.test(value)) {
return `href=${quote}../../packages/amis-ui/scss/themes/${RegExp.$1}.scss${quote}`;
} else if (/^amis\/lib\/helper\.css$/.test(value)) {
return `href=${quote}../../packages/amis-ui/scss/helper.scss${quote}`;
} else if (
/^(?:amis|amis\-core|amis\-formula|amis\-ui|office\-viewer)/.test(
value
)
) {
return `href=${quote}../../packages/${value}${quote}`;
} else if (value[0] !== '.' && value[0] !== '/') {
return `href=${quote}../../node_modules/${value}${quote}`;
}
return _;
});
const script = '<script type="text/javascript">';
const idx = html.lastIndexOf(script);
if (~idx) {
html =
html.substring(0, idx) +
'<script type="module">' +
html.substring(idx + script.length);
}
html = html.replace(
"amis.require(['./mobile.jsx'], function (app)",
"import('./mobile.jsx').then(function (app)"
);
}
return html;
}
};
}

73
scripts/upload2cdn.js Normal file
View File

@@ -0,0 +1,73 @@
const bosSDK = require('bce-sdk-js');
const fs = require('fs');
const walk = require('fs-walk');
const path = require('path');
const ak = process.env.BOS_AK;
const sk = process.env.BOS_SK;
if (!ak || !sk) {
console.error('ak and sk is required for deploy cdn!');
process.exit(1);
}
const config = {
credentials: {
ak,
sk
},
endpoint: 'https://bj.bcebos.com',
bucket: 'bce-cdn'
};
const client = new bosSDK.BosClient(config);
const bucketName = config.bucket || 'bce-cdn';
const prefix = 'fex/';
async function main() {
const folder = 'gh-pages';
const productName = 'amis-gh-pages';
if (!fs.existsSync(folder)) {
throw new Error('文件夹不存在');
}
if (!productName) {
throw new Error('请指定产品名称');
}
const promises = [];
walk.walkSync(folder, (basedir, filename, stat) => {
if (stat.isDirectory() || /\.html$/.test(filename)) {
return;
}
const relativePath = path.relative(folder, basedir);
// 隐藏文件跳过。
if (/^\./.test(relativePath)) {
return;
}
const objectName = path.join(prefix, productName, relativePath, filename);
promises.push(() =>
client
.putObjectFromFile(bucketName, objectName, path.join(basedir, filename))
.then(
() => console.log(` ==> ${objectName}`),
res => console.error(res)
)
);
});
if (!promises.length) {
console.log('Nothing need to upload');
}
return promises.reduce((preview, current) => {
return preview.then(current);
}, Promise.resolve());
}
main()
.then(() => console.log('Done!'))
.catch(e => console.error(e.message, e.stack));

83
scripts/utils.sh Normal file
View File

@@ -0,0 +1,83 @@
#!/bin/bash
render_option() {
for index in "${!choices[@]}"; do
if [ $index -eq $1 ]; then
printf "\033[31m> ${choices[$index]}\033[0m\n" # 高亮显示选中的选项
else
echo " ${choices[$index]}"
fi
done
}
clear_screen() {
local len=$1
echo "\033[${len}A\033[K"
for ((i=1;i<$len;i++)); do
echo "\033[K"
done
echo "\033[$((${len} + 1))A"
}
select_option() {
choices=("$@") # 将选项数组声明为全局变量
local selected=0 # 初始化选择索引
local choices_lenght=${#choices[@]} # 获取选项数组长度
render_option $selected
while true; do
read -n1 -s key # 读取单个按键并保持输入的隐私
case "$key" in
A) # 上箭头
if [ $selected -gt 0 ]; then
selected=$((selected - 1))
else
selected=$(($choices_lenght - 1))
fi
clear_screen $choices_lenght
render_option $selected
;;
B) # 下箭头
if [ $selected -lt $(($choices_lenght - 1)) ]; then
selected=$((selected + 1))
else
selected=0
fi
clear_screen $choices_lenght
render_option $selected
;;
"") # 回车键
break
;;
esac
done
# 打印最终结果日志
selected_option="${choices[$selected]}"
}
npm_version() {
local current_version=$1
local tag=$2
IFS='.-' read -r major minor patch pre_release pre_release_version <<< "$current_version"
if [ $tag = $pre_release ]; then
((pre_release_version++))
new_version="$major.$minor.$patch-$tag.$pre_release_version"
else
((patch++))
new_version="$major.$minor.$patch"
fi
echo "新版本为:$new_version"
}
npm_get_tag() {
IFS='.-' read -r major minor patch pre_release pre_release_version <<< "$1"
if [ "$pre_release" ]; then
npm_tag=$pre_release
else
npm_tag="latest"
fi
echo "发布tag为: $npm_tag"
}

View File

@@ -0,0 +1,34 @@
/**
* 用于查找某个变量在 cxd-variables 里但不在 properties.scss 里的情况
*/
const fs = require('fs');
const path = require('path');
const cxdVariables = fs.readFileSync(
path.join(__dirname, '..', 'scss', 'themes', '_cxd-variables.scss'),
{encoding: 'utf8'}
);
const commonVariables = fs.readFileSync(
path.join(__dirname, '..', 'scss', '_properties.scss'),
{encoding: 'utf8'}
);
const cxdVariableSet = new Set();
cxdVariables.match(/\-\-[\-a-zA-Z0-9]+/g).forEach(function (variable) {
cxdVariableSet.add(variable);
});
const commonVariableSet = new Set();
commonVariables.match(/\-\-[\-a-zA-Z0-9]+/g).forEach(function (variable) {
commonVariableSet.add(variable);
});
for (const variable of cxdVariableSet) {
if (!commonVariableSet.has(variable)) {
console.log(variable);
}
}