#!/usr/bin/env python3
# webapp.py - 手写数字识别系统（新极简主义风格 + 双语音播报）
# 依赖: flask opencv-python torch torchvision edge-tts numpy pillow
import os
import cv2
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import numpy as np
import subprocess
import tempfile
import base64
import threading
import asyncio
import edge_tts
from flask import Flask, Response, render_template_string, jsonify, request
# ---------------------------- 1. 神经网络模型（与 model.py 一致） ----------------------------
class NeuralNetwork(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv2d(1, 32, 3, 1)
self.conv2 = nn.Conv2d(32, 64, 3, 1)
self.fc1 = nn.Linear(in_features=64 * 5 * 5, out_features=128)
self.fc2 = nn.Linear(in_features=128, out_features=10)
def forward(self, x):
x = F.relu(self.conv1(x))
x = F.max_pool2d(x, kernel_size=2)
x = F.relu(self.conv2(x))
x = F.max_pool2d(x, kernel_size=2)
x = x.view(-1, 64 * 5 * 5)
x = F.relu(self.fc1(x))
x = self.fc2(x)
return x
# ---------------------------- 2. 摄像头处理（带黄色方框） ----------------------------
class Camera:
def __init__(self, camera_id=0):
self.cap = cv2.VideoCapture(camera_id)
if not self.cap.isOpened():
raise RuntimeError("无法打开摄像头")
ret, frame = self.cap.read()
if not ret:
raise RuntimeError("无法读取摄像头画面")
self.height, self.width = frame.shape[:2]
side = int(min(self.width, self.height) * 0.8)
self.roi_x = (self.width - side) // 2
self.roi_y = (self.height - side) // 2
self.roi_size = side
self.lock = threading.Lock()
self.latest_frame = frame
def get_frame_with_bbox(self):
with self.lock:
ret, frame = self.cap.read()
if not ret:
frame = self.latest_frame
else:
self.latest_frame = frame
cv2.rectangle(
frame,
(self.roi_x, self.roi_y),
(self.roi_x + self.roi_size, self.roi_y + self.roi_size),
(0, 255, 255),
3,
)
ret, jpeg = cv2.imencode(".jpg", frame)
return jpeg.tobytes()
def capture_roi(self):
with self.lock:
frame = self.latest_frame.copy()
roi = frame[
self.roi_y : self.roi_y + self.roi_size,
self.roi_x : self.roi_x + self.roi_size,
]
return roi
def release(self):
self.cap.release()
# ---------------------------- 3. 数字识别器（预处理 + 推理 + 置信度） ----------------------------
class DigitRecognizer:
def __init__(self, weights_path="mnist.pth"):
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.model = NeuralNetwork().to(self.device)
if not os.path.exists(weights_path):
raise FileNotFoundError(f"模型文件 {weights_path} 不存在")
self.model.load_state_dict(torch.load(weights_path, map_location=self.device))
self.model.eval()
def preprocess(self, roi_bgr):
gray = cv2.cvtColor(roi_bgr, cv2.COLOR_BGR2GRAY)
gray_inv = 255 - gray
_, binary = cv2.threshold(gray_inv, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
contours, _ = cv2.findContours(
binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
)
if contours:
x_min, y_min = 9999, 9999
x_max, y_max = 0, 0
for cnt in contours:
x, y, w, h = cv2.boundingRect(cnt)
x_min = min(x_min, x)
y_min = min(y_min, y)
x_max = max(x_max, x + w)
y_max = max(y_max, y + h)
digit_roi = binary[y_min:y_max, x_min:x_max]
else:
digit_roi = binary
if digit_roi.size == 0:
digit_roi = np.zeros((20, 20), dtype=np.uint8)
else:
digit_roi = cv2.resize(digit_roi, (20, 20), interpolation=cv2.INTER_AREA)
final_img = np.zeros((28, 28), dtype=np.uint8)
x_offset = (28 - 20) // 2
y_offset = (28 - 20) // 2
final_img[y_offset : y_offset + 20, x_offset : x_offset + 20] = digit_roi
transform = torchvision.transforms.Compose(
[
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize((0.1307,), (0.3081,)),
]
)
tensor = transform(final_img).unsqueeze(0).to(self.device)
return tensor, final_img
def predict(self, roi_bgr):
tensor, processed_img = self.preprocess(roi_bgr)
with torch.no_grad():
output = self.model(tensor)
probs = F.softmax(output, dim=1)
pred = output.argmax(dim=1).item()
prob_list = probs.squeeze().cpu().numpy().tolist()
return pred, prob_list, processed_img
# ---------------------------- 4. 语音播报（开发板本地播放 + 返回音频） ----------------------------
"""
VOICES = [
"zh-CN-XiaoxiaoNeural",
"zh-CN-XiaoyiNeural",
"zh-CN-YunjianNeural",
"zh-CN-YunxiNeural",
"zh-CN-YunxiaNeural",
"zh-CN-YunyangNeural",
"zh-CN-liaoning-XiaobeiNeural", # 辽宁方言
"zh-CN-shaanxi-XiaoniNeural", # 陕西方言
]
"""
async def speak_and_play_local(text: str, voice: str = "zh-CN-shaanxi-XiaoniNeural"):
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f:
tmp_file = f.name
try:
await edge_tts.Communicate(text, voice).save(tmp_file)
try:
subprocess.Popen(
["ffplay", "-nodisp", "-autoexit", tmp_file],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
except FileNotFoundError:
subprocess.Popen(
["mpg123", "-q", tmp_file],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
return tmp_file
except Exception as e:
print(f"语音播报失败: {e}")
if os.path.exists(tmp_file):
os.unlink(tmp_file)
return None
def cleanup_audio_file(filepath):
if filepath and os.path.exists(filepath):
os.unlink(filepath)
# ---------------------------- 5. Flask Web 服务（含新极简主义前端） ----------------------------
app = Flask(__name__)
camera = None
recognizer = None
def get_camera():
global camera
if camera is None:
camera = Camera(camera_id=0)
return camera
def get_recognizer():
global recognizer
if recognizer is None:
recognizer = DigitRecognizer("mnist.pth")
return recognizer
# 内嵌 HTML/CSS/JS (单色调柔和粉彩 + 卡片式 + 新极简主义)
HTML_TEMPLATE = """
手写数字识别 | 新极简主义
📷 拍摄区域
"""
@app.route("/")
def index():
return render_template_string(HTML_TEMPLATE)
@app.route("/video_feed")
def video_feed():
def generate():
cam = get_camera()
while True:
frame = cam.get_frame_with_bbox()
yield (
b"--frame\r\n" b"Content-Type: image/jpeg\r\n\r\n" + frame + b"\r\n\r\n"
)
return Response(generate(), mimetype="multipart/x-mixed-replace; boundary=frame")
@app.route("/capture", methods=["POST"])
def capture():
cam = get_camera()
roi_img = cam.capture_roi()
_, buffer = cv2.imencode(".jpg", roi_img)
img_base64 = base64.b64encode(buffer).decode("utf-8")
app.config["LATEST_ROI"] = roi_img
return jsonify({"image": img_base64})
@app.route("/recognize", methods=["POST"])
def recognize():
roi_img = app.config.get("LATEST_ROI")
if roi_img is None:
return jsonify({"error": "请先拍照"}), 400
rec = get_recognizer()
pred, prob_list, processed_img = rec.predict(roi_img)
_, buffer = cv2.imencode(".png", processed_img)
processed_base64 = base64.b64encode(buffer).decode("utf-8")
text = f"识别到的数字是{pred}"
tts_mp3_data = None
try:
audio_file = asyncio.run(speak_and_play_local(text))
if audio_file and os.path.exists(audio_file):
with open(audio_file, "rb") as f:
tts_mp3_data = base64.b64encode(f.read()).decode("utf-8")
threading.Timer(5.0, lambda: cleanup_audio_file(audio_file)).start()
except Exception as e:
print(f"语音播报异常: {e}")
return jsonify(
{
"digit": pred,
"probabilities": prob_list,
"processed_image": processed_base64,
"audio_base64": tts_mp3_data,
}
)
if __name__ == "__main__":
if not os.path.exists("mnist.pth"):
print("错误：未找到 mnist.pth，请先训练模型。")
exit(1)
# 检查播放器
try:
subprocess.run(["ffplay", "-version"], capture_output=True, check=True)
print("✅ 语音播放器 ffplay 已就绪")
except:
try:
subprocess.run(["mpg123", "-V"], capture_output=True, check=True)
print("✅ 语音播放器 mpg123 已就绪")
except:
print("⚠️ 未找到播放器，开发板可能无声。安装: sudo apt install ffmpeg")
app.run(host="0.0.0.0", port=5000, debug=False, threaded=True)
# 1、USB摄像头，接在开发板上
# 2、在连接开发板的个人电脑上，通过网页方式访问开发板
# 3、在网页上，可以使用接在开发板上的USB摄像头拍照，对手写的数字拍照
# 4、然后让开发板上的程序做识别，并反馈识别结果到个人电脑的浏览器上
# 5、浏览器有个窗口，可以看到摄像头的画面。
# 6、拍摄窗口中，加个正方形的黄框，尽可能大。仅拍摄黄框内的画面。
# 7、拍照和识别，拆成2个按钮。拍照后，显示被拍到的照片；识别，对照片识别
# 8、处理为 28*28 的图片，送给推理程序识别的，也要显示在Web页面上
# 9、如果拍摄到3，识别到3， 0-9共10个数字的置性度，也要显示的web页面上
# 10、使用接在开发板上的喇叭加个语音播报，比如识别到的数字是3。web界面上播放也保留。
# 11、输出一个样例文件，不要分多个文件
# 12、web界面要有高级感，参照如下原则：
# - web界面的颜色：monochromatic muted pastel
# - web界面的layout：card based design with layered elements
# - web界面的风格：Neo-minumalism
# - web界面的设计哲学：approachable sophistication
# 参考程序如下：
# 1、语音播报怎么写，请参考：语音try_tts.py
# 2、模型训练程序：train.py
# 3、模型定义：model.py