feat: 完成龙虾记忆同步系统

后端:
- Django + DRF
- PostgreSQL 数据库
- 文件扫描服务
- 差异检查服务
- 完整 REST API

前端:
- React + Ant Design
- 文件树展示
- 差异对比
- API 客户端封装

部署:
- Docker Compose
- 后端 Dockerfile
- 前端 Dockerfile
- 一键启动脚本

功能:
- 扫描龙虾记忆文件
- 检查文件差异
- 双向同步(本地 <-> 数据库)
- 版本历史
- 统计信息
This commit is contained in:
道童
2026-04-05 12:04:13 +00:00
commit f176e2d818
24 changed files with 1621 additions and 0 deletions

17
backend/Dockerfile Normal file
View File

@@ -0,0 +1,17 @@
FROM python:3.11-slim
WORKDIR /app
# 安装依赖
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# 复制代码
COPY . .
# 收集静态文件
RUN python manage.py collectstatic --noinput
EXPOSE 8087
CMD ["python", "manage.py", "runserver", "0.0.0.0:8087"]

23
backend/manage.py Normal file
View File

@@ -0,0 +1,23 @@
# Django manage.py
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
"""Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'memory_sync.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,61 @@
from django.db import models
from django.core.validators import FileExtensionValidator
import hashlib
class LobsterMemory(models.Model):
"""龙虾记忆文件模型"""
STATUS_CHOICES = [
('consistent', '一致'),
('local_newer', '本地更新'),
('db_newer', '数据库更新'),
('conflict', '冲突'),
]
lobster_id = models.CharField(max_length=50, help_text='龙虾ID')
file_path = models.CharField(max_length=500, help_text='文件相对路径')
content = models.TextField(help_text='文件内容')
hash = models.CharField(max_length=64, help_text='SHA256哈希')
status = models.CharField(
max_length=20,
choices=STATUS_CHOICES,
default='consistent',
help_text='同步状态'
)
version = models.IntegerField(default=1, help_text='版本号')
size = models.IntegerField(default=0, help_text='文件大小(字节)')
created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
updated_at = models.DateTimeField(auto_now=True, help_text='更新时间')
class Meta:
db_table = 'lobster_memory'
unique_together = ('lobster_id', 'file_path', 'version')
ordering = ['-updated_at']
indexes = [
models.Index(fields=['lobster_id', 'file_path']),
models.Index(fields=['status']),
models.Index(fields=['updated_at']),
]
def __str__(self):
return f"{self.lobster_id}/{self.file_path} (v{self.version})"
def compute_hash(self, content):
"""计算SHA256哈希"""
return hashlib.sha256(content.encode('utf-8')).hexdigest()
def save(self, *args, **kwargs):
"""保存时自动计算哈希和大小"""
if self.content:
self.hash = self.compute_hash(self.content)
self.size = len(self.content.encode('utf-8'))
super().save(*args, **kwargs)

View File

@@ -0,0 +1,35 @@
from rest_framework import serializers
from .models import LobsterMemory
class LobsterMemorySerializer(serializers.ModelSerializer):
"""龙虾记忆序列化器"""
class Meta:
model = LobsterMemory
fields = [
'id',
'lobster_id',
'file_path',
'content',
'hash',
'status',
'version',
'size',
'created_at',
'updated_at',
]
read_only_fields = ['id', 'created_at', 'updated_at']
class FileDiffSerializer(serializers.Serializer):
"""文件差异序列化器"""
file_path = serializers.CharField()
lobster_id = serializers.CharField()
local_content = serializers.CharField(required=False)
db_content = serializers.CharField(required=False)
local_hash = serializers.CharField(required=False)
db_hash = serializers.CharField(required=False)
status = serializers.CharField()
message = serializers.CharField(required=False)

View File

@@ -0,0 +1,224 @@
import os
import hashlib
from pathlib import Path
from typing import List, Dict, Tuple
from django.conf import settings
class FileScanner:
"""文件扫描器"""
def __init__(self):
self.base_dir = Path(settings.LOBSTER_MEMORY_BASE)
self.supported_extensions = settings.SUPPORTED_EXTENSIONS
def scan_directory(self, lobster_id: str = None) -> List[Dict]:
"""
扫描目录,返回所有文件信息
Args:
lobster_id: 龙虾ID可选
Returns:
文件信息列表
"""
if not self.base_dir.exists():
return []
files = []
for file_path in self.base_dir.rglob('*'):
if file_path.is_file() and file_path.suffix in self.supported_extensions:
try:
relative_path = file_path.relative_to(self.base_dir)
content = file_path.read_text(encoding='utf-8', errors='ignore')
file_hash = self.compute_hash(content)
files.append({
'file_path': str(relative_path),
'full_path': str(file_path),
'content': content,
'hash': file_hash,
'size': file_path.stat().st_size,
'lobster_id': lobster_id or 'unknown',
})
except Exception as e:
print(f"Error reading {file_path}: {e}")
return files
def get_file_content(self, file_path: str) -> Tuple[str, str]:
"""
获取文件内容和哈希
Args:
file_path: 相对路径
Returns:
(content, hash)
"""
full_path = self.base_dir / file_path
if not full_path.exists():
raise FileNotFoundError(f"File not found: {file_path}")
content = full_path.read_text(encoding='utf-8', errors='ignore')
file_hash = self.compute_hash(content)
return content, file_hash
def write_file(self, file_path: str, content: str):
"""
写入文件
Args:
file_path: 相对路径
content: 文件内容
"""
full_path = self.base_dir / file_path
# 确保目录存在
full_path.parent.mkdir(parents=True, exist_ok=True)
# 写入文件
full_path.write_text(content, encoding='utf-8')
def compute_hash(self, content: str) -> str:
"""
计算SHA256哈希
Args:
content: 文件内容
Returns:
哈希值
"""
return hashlib.sha256(content.encode('utf-8')).hexdigest()
def get_file_tree(self, lobster_id: str = None) -> Dict:
"""
获取文件树结构
Args:
lobster_id: 龙虾ID
Returns:
文件树字典
"""
files = self.scan_directory(lobster_id)
tree = {}
for file_info in files:
parts = Path(file_info['file_path']).parts
current = tree
for part in parts[:-1]:
if part not in current:
current[part] = {}
current = current[part]
filename = parts[-1]
current[filename] = file_info
return tree
class DiffChecker:
"""差异检查器"""
def __init__(self):
self.scanner = FileScanner()
def check_sync_status(self, local_files: List[Dict], db_files: List[Dict]) -> Dict:
"""
检查同步状态
Args:
local_files: 本地文件列表
db_files: 数据库文件列表
Returns:
同步状态字典
"""
local_map = {f['file_path']: f for f in local_files}
db_map = {f['file_path']: f for f in db_files}
results = {
'consistent': [],
'local_newer': [],
'db_newer': [],
'conflict': [],
'local_only': [],
'db_only': [],
}
all_paths = set(local_map.keys()) | set(db_map.keys())
for path in all_paths:
local = local_map.get(path)
db = db_map.get(path)
if local and db:
# 两边都存在
if local['hash'] == db['hash']:
results['consistent'].append({
'file_path': path,
'status': 'consistent'
})
else:
# 比较更新时间
local_time = db.get('updated_at') if db else None
if local_time:
# 数据库有更新时间,比较
if local['hash'] != db['hash']:
results['conflict'].append({
'file_path': path,
'status': 'conflict',
'local_hash': local['hash'],
'db_hash': db['hash']
})
else:
# 无法判断,标记为冲突
results['conflict'].append({
'file_path': path,
'status': 'conflict',
'local_hash': local['hash'],
'db_hash': db['hash']
})
elif local and not db:
# 只有本地
results['local_only'].append({
'file_path': path,
'status': 'local_only'
})
elif not local and db:
# 只有数据库
results['db_only'].append({
'file_path': path,
'status': 'db_only'
})
return results
def get_file_diff(self, local_content: str, db_content: str) -> Dict:
"""
获取文件差异(简单版)
Args:
local_content: 本地内容
db_content: 数据库内容
Returns:
差异信息
"""
# 这里可以使用 difflib 或其他差异库
# 简单实现,后续可以用 react-diff-viewer 在前端显示
return {
'local_lines': local_content.split('\n'),
'db_lines': db_content.split('\n'),
'has_diff': local_content != db_content
}

View File

@@ -0,0 +1,24 @@
from django.urls import path
from . import views
urlpatterns = [
# 扫描相关
path('scan/', views.scan_files, name='scan_files'),
path('tree/', views.get_file_tree, name='get_file_tree'),
# 同步状态
path('status/', views.check_sync_status, name='check_sync_status'),
# 差异对比
path('diff/', views.get_file_diff, name='get_file_diff'),
# 同步操作
path('sync/db/', views.sync_to_db, name='sync_to_db'),
path('sync/local/', views.sync_to_local, name='sync_to_local'),
# 版本历史
path('versions/', views.get_versions, name='get_versions'),
# 统计信息
path('stats/', views.get_stats, name='get_stats'),
]

303
backend/memory_app/views.py Normal file
View File

@@ -0,0 +1,303 @@
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework import status
from .models import LobsterMemory
from .serializers import LobsterMemorySerializer, FileDiffSerializer
from .services import FileScanner, DiffChecker
import json
@api_view(['GET'])
def scan_files(request):
"""
扫描本地文件
"""
lobster_id = request.query_params.get('lobster_id', 'daotong')
scanner = FileScanner()
files = scanner.scan_directory(lobster_id)
return Response({
'success': True,
'data': files,
'total': len(files)
})
@api_view(['GET'])
def get_file_tree(request):
"""
获取文件树
"""
lobster_id = request.query_params.get('lobster_id', 'daotong')
scanner = FileScanner()
tree = scanner.get_file_tree(lobster_id)
return Response({
'success': True,
'data': tree
})
@api_view(['GET'])
def check_sync_status(request):
"""
检查同步状态
"""
lobster_id = request.query_params.get('lobster_id', 'daotong')
# 获取本地文件
scanner = FileScanner()
local_files = scanner.scan_directory(lobster_id)
# 获取数据库文件
db_files = list(LobsterMemory.objects.filter(
lobster_id=lobster_id
).values('file_path', 'hash', 'version', 'updated_at'))
# 检查同步状态
checker = DiffChecker()
sync_status = checker.check_sync_status(local_files, db_files)
return Response({
'success': True,
'data': sync_status
})
@api_view(['GET'])
def get_file_diff(request):
"""
获取文件差异
"""
file_path = request.query_params.get('file_path')
lobster_id = request.query_params.get('lobster_id', 'daotong')
if not file_path:
return Response({
'success': False,
'error': 'file_path is required'
}, status=status.HTTP_400_BAD_REQUEST)
scanner = FileScanner()
# 获取本地内容
try:
local_content, local_hash = scanner.get_file_content(file_path)
except FileNotFoundError:
local_content = None
local_hash = None
# 获取数据库内容
try:
db_record = LobsterMemory.objects.filter(
lobster_id=lobster_id,
file_path=file_path
).order_by('-version').first()
if db_record:
db_content = db_record.content
db_hash = db_record.hash
else:
db_content = None
db_hash = None
except Exception as e:
return Response({
'success': False,
'error': str(e)
}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 获取差异
checker = DiffChecker()
if local_content and db_content:
diff = checker.get_file_diff(local_content, db_content)
else:
diff = {
'local_lines': local_content.split('\n') if local_content else [],
'db_lines': db_content.split('\n') if db_content else [],
'has_diff': local_content != db_content
}
# 确定状态
if local_hash == db_hash:
status = 'consistent'
elif local_hash and not db_hash:
status = 'local_newer'
elif not local_hash and db_hash:
status = 'db_newer'
else:
status = 'conflict'
return Response({
'success': True,
'data': {
'file_path': file_path,
'lobster_id': lobster_id,
'local_content': local_content,
'db_content': db_content,
'local_hash': local_hash,
'db_hash': db_hash,
'status': status,
'diff': diff
}
})
@api_view(['POST'])
def sync_to_db(request):
"""
同步到数据库
"""
lobster_id = request.data.get('lobster_id', 'daotong')
file_path = request.data.get('file_path')
if not file_path:
return Response({
'success': False,
'error': 'file_path is required'
}, status=status.HTTP_400_BAD_REQUEST)
scanner = FileScanner()
try:
# 读取本地文件
content, file_hash = scanner.get_file_content(file_path)
# 查找现有记录
existing = LobsterMemory.objects.filter(
lobster_id=lobster_id,
file_path=file_path
).order_by('-version').first()
if existing:
# 创建新版本
new_version = existing.version + 1
else:
new_version = 1
# 创建新记录
record = LobsterMemory.objects.create(
lobster_id=lobster_id,
file_path=file_path,
content=content,
hash=file_hash,
status='consistent',
version=new_version,
)
return Response({
'success': True,
'message': '已同步到数据库',
'data': LobsterMemorySerializer(record).data
})
except Exception as e:
return Response({
'success': False,
'error': str(e)
}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@api_view(['POST'])
def sync_to_local(request):
"""
同步到本地
"""
lobster_id = request.data.get('lobster_id', 'daotong')
file_path = request.data.get('file_path')
if not file_path:
return Response({
'success': False,
'error': 'file_path is required'
}, status=status.HTTP_400_BAD_REQUEST)
scanner = FileScanner()
try:
# 从数据库获取最新版本
db_record = LobsterMemory.objects.filter(
lobster_id=lobster_id,
file_path=file_path
).order_by('-version').first()
if not db_record:
return Response({
'success': False,
'error': 'File not found in database'
}, status=status.HTTP_404_NOT_FOUND)
# 写入本地文件
scanner.write_file(file_path, db_record.content)
return Response({
'success': True,
'message': '已同步到本地',
'data': LobsterMemorySerializer(db_record).data
})
except Exception as e:
return Response({
'success': False,
'error': str(e)
}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@api_view(['GET'])
def get_versions(request):
"""
获取文件的所有版本
"""
file_path = request.query_params.get('file_path')
lobster_id = request.query_params.get('lobster_id', 'daotong')
if not file_path:
return Response({
'success': False,
'error': 'file_path is required'
}, status=status.HTTP_400_BAD_REQUEST)
versions = LobsterMemory.objects.filter(
lobster_id=lobster_id,
file_path=file_path
).order_by('-version')
return Response({
'success': True,
'data': LobsterMemorySerializer(versions, many=True).data
})
@api_view(['GET'])
def get_stats(request):
"""
获取统计信息
"""
lobster_id = request.query_params.get('lobster_id', 'daotong')
total_files = LobsterMemory.objects.filter(lobster_id=lobster_id).count()
status_counts = {}
for status_choice, _ in LobsterMemory.STATUS_CHOICES:
count = LobsterMemory.objects.filter(
lobster_id=lobster_id,
status=status_choice
).count()
status_counts[status_choice] = count
# 获取总大小
from django.db.models import Sum
total_size = LobsterMemory.objects.filter(
lobster_id=lobster_id
).aggregate(total=Sum('size'))['total'] or 0
return Response({
'success': True,
'data': {
'total_files': total_files,
'status_counts': status_counts,
'total_size': total_size,
'total_size_mb': round(total_size / 1024 / 1024, 2)
}
})

View File

@@ -0,0 +1,101 @@
"""
Django settings for memory_sync project.
"""
from pathlib import Path
import os
BASE_DIR = Path(__file__).resolve().parent.parent
SECRET_KEY = 'django-insecure-dev-key-change-in-production'
DEBUG = True
ALLOWED_HOSTS = ['*']
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'rest_framework',
'corsheaders',
'memory_app',
]
MIDDLEWARE = [
'corsheaders.middleware.CorsMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'memory_sync.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'memory_sync.wsgi.application'
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': os.getenv('DB_NAME', 'lobster_memory'),
'USER': os.getenv('DB_USER', 'postgres'),
'PASSWORD': os.getenv('DB_PASSWORD', 'postgres'),
'HOST': os.getenv('DB_HOST', 'localhost'),
'PORT': os.getenv('DB_PORT', '5432'),
}
}
AUTH_PASSWORD_VALIDATORS = []
LANGUAGE_CODE = 'zh-hans'
TIME_ZONE = 'Asia/Shanghai'
USE_I18N = True
USE_TZ = True
STATIC_URL = 'static/'
STATIC_ROOT = BASE_DIR / 'staticfiles'
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
# REST Framework
REST_FRAMEWORK = {
'DEFAULT_PERMISSION_CLASSES': [
'rest_framework.permissions.AllowAny',
],
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
'PAGE_SIZE': 100,
}
# CORS
CORS_ALLOW_ALL_ORIGINS = True
# 龙虾记忆目录
LOBSTER_MEMORY_BASE = os.getenv('LOBSTER_MEMORY_BASE', '/home/node/.openclaw/workspace/daotong')
# 支持的文件扩展名
SUPPORTED_EXTENSIONS = ['.md', '.txt', '.json', '.py', '.js', '.yaml', '.yml']

View File

@@ -0,0 +1,7 @@
from django.contrib import admin
from django.urls import path, include
urlpatterns = [
path('admin/', admin.site.urls),
path('api/', include('memory_app.urls')),
]

View File

@@ -0,0 +1,11 @@
"""
WSGI config for memory_sync project.
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'memory_sync.settings')
application = get_wsgi_application()

5
backend/requirements.txt Normal file
View File

@@ -0,0 +1,5 @@
Django>=4.2.0,<5.0.0
djangorestframework>=3.14.0
django-cors-headers>=4.0.0
psycopg2-binary>=2.9.0
python-dotenv>=1.0.0