From 787b4d6f05de2ae0fe1e0fa5d326126818d6da7f Mon Sep 17 00:00:00 2001 From: wang Date: Tue, 21 Jul 2026 18:54:45 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96ftp=E8=AF=AD=E9=9F=B3?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=8B=89=E5=8F=96=E6=9C=8D=E5=8A=A1=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/application-external.yml | 46 ++-- .../dataserviceyy/service/FtpSyncService.java | 30 +- .../dataserviceyy/util/FilePathUtil.java | 70 +---- .../dataserviceyy/util/FileUploadUtil.java | 257 ++++++------------ src/main/resources/application.yml | 5 +- 5 files changed, 104 insertions(+), 304 deletions(-) diff --git a/config/application-external.yml b/config/application-external.yml index 1dc06a9..f0b78a0 100644 --- a/config/application-external.yml +++ b/config/application-external.yml @@ -36,33 +36,33 @@ ftp-sync: # OSS 文件上传配置 oss: - base-url: http://127.0.0.1:9090 - upload-url: http://127.0.0.1:9090/apiOss/oss/noAuthFileUploadSingle - # 应用编码可以不写 - appcode: dataservice-yy - # 应用ID（部署时改为真实值） - appid: your_appid_here - # 应用密钥（部署时改为真实值） - appsecret: your_appsecret_here + base-url: http://53.1.211.7/apiOss - # 当前部署的地市FTP配置（每个地市独立部署一份） + # 地市FTP配置（每个地市独立部署一份，只配当前地市即可） cities: - - city-code: "340100" - city-name: "合肥" + # ===== 淮南 ===== + - city-code: "340400" + city-name: "淮南" ftp-host: 10.126.129.7 ftp-port: 9979 - ftp-username: ftpuser - ftp-password: ftppass - ftp-source-dir: /340100/record/ - ftp-record-dir: /340100/recordfile/ - ftp-archive-dir: /340100/processed/ - # 电话区号前缀（处理主被叫号码时去除） - # - 配正确区号：号码以这个区号开头则去除 - # - 配不存在的区号（如 "0000"）：号码匹配不上，保留原始号码 - # - 留空：保留原始号码 - # 合肥: 0551, 芜湖: 0553, 蚌埠: 0552, 淮南: 0554, 马鞍山: 0555 - # 当前配置 "0000" 表示不去除区号（保留原始号码） - phone-area-code: "0000" + ftp-username: yyfile + ftp-password: yyfile + ftp-source-dir: /voice_record/ + ftp-record-dir: /rec/ + ftp-archive-dir: /voice_record/processed/ + phone-area-code: "0554" + + # ===== 蚌埠 ===== + - city-code: "340300" + city-name: "蚌埠" + ftp-host: 10.126.54.104 + ftp-port: 9989 + ftp-username: ftpadmin + ftp-password: A3833508q + ftp-source-dir: /voice_record/ + ftp-record-dir: /rec/ + ftp-archive-dir: /voice_record/processed/ + phone-area-code: "0552" # ==================== 服务端口 ==================== server: diff --git a/src/main/java/com/threecloud/dataserviceyy/service/FtpSyncService.java b/src/main/java/com/threecloud/dataserviceyy/service/FtpSyncService.java index 8bfed53..3f52d61 100644 --- a/src/main/java/com/threecloud/dataserviceyy/service/FtpSyncService.java +++ b/src/main/java/com/threecloud/dataserviceyy/service/FtpSyncService.java @@ -312,25 +312,12 @@ public class FtpSyncService { logger.info("【FTP行】准备下载录音: {}/{}", recordDir, remoteRecordPath); byte[] recordBytes = FtpUtil.downloadFile(ftp, remoteRecordPath); logger.info("【FTP行】下载录音: {} ({} 字节)", recordFileName, recordBytes.length); - - // 为了方便您测试验证链路，这里加上一段代码，把从FTP读到的录音直接存在本地硬盘 - try { - java.nio.file.Path testSaveDir = java.nio.file.Paths.get("download_test_records", dateStr); - if (!java.nio.file.Files.exists(testSaveDir)) { - java.nio.file.Files.createDirectories(testSaveDir); - } - java.nio.file.Path testSaveFile = testSaveDir.resolve(recordFileName); - java.nio.file.Files.write(testSaveFile, recordBytes); - logger.info("【FTP测试】已成功将录音保存到本地硬盘: {}", testSaveFile.toAbsolutePath()); - } catch (Exception e) { - logger.error("【FTP测试】保存本地测试文件失败: {}", e.getMessage()); - } // 下载完成后切回txt目录，继续处理下一个文件 FtpUtil.changeDir(ftp, city.getFtpSourceDir()); // 上传到OSS（获取可访问的URL） - String ossUrl = fileUploadUtil.uploadWav(city.getCityName(), ossPath, recordFileName, recordBytes); + String ossUrl = fileUploadUtil.uploadWav(ossPath, recordFileName, recordBytes); logger.info("【FTP行】上传OSS: {}", ossUrl); // 构建实体对象并入库 @@ -340,21 +327,6 @@ public class FtpSyncService { existingIds.add(callRecordId); // 防止同批内重复（本次循环内） logger.info("【FTP行】保存成功: id={}, callRecordId={}", rec.getId(), callRecordId); - // 将成功信息记录到本地文件中，方便排查和核对 - try { - java.nio.file.Path logDir = java.nio.file.Paths.get("logs"); - if (!java.nio.file.Files.exists(logDir)) { - java.nio.file.Files.createDirectories(logDir); - } - java.nio.file.Path logFile = logDir.resolve("upload_success.log"); - String logLine = String.format("[%s] 成功同步录音: 流水号=%s, 文件名=%s, OSS地址=%s\n", - DateUtil.formatDate(new Date(), "yyyy-MM-dd HH:mm:ss"), callRecordId, recordFileName, ossUrl); - java.nio.file.Files.write(logFile, logLine.getBytes("UTF-8"), - java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND); - } catch (Exception e) { - logger.error("记录成功日志到文件失败", e); - } - return true; } diff --git a/src/main/java/com/threecloud/dataserviceyy/util/FilePathUtil.java b/src/main/java/com/threecloud/dataserviceyy/util/FilePathUtil.java index 3b28145..cbef915 100644 --- a/src/main/java/com/threecloud/dataserviceyy/util/FilePathUtil.java +++ b/src/main/java/com/threecloud/dataserviceyy/util/FilePathUtil.java @@ -1,52 +1,16 @@ package com.threecloud.dataserviceyy.util; -import java.io.File; import java.text.SimpleDateFormat; import java.util.Date; /** * 文件路径工具类 - * 统一管理本地文件存储路径生成逻辑 + * 统一管理OSS文件存储路径生成逻辑 */ public class FilePathUtil { - /** - * 日期格式：yyyyMMdd - */ private static final String DATE_FORMAT = "yyyyMMdd"; - /** - * 生成本地文件存储路径（按地市分文件夹） - * - * 路径格式: {basePath}/{cityCode}/{date}/{uuid}/{fileName} - * 示例: ./vaa-recordings/340100/20240101/uuid-xxx/xxx.wav - * - * @param basePath 基础路径（如 ./vaa-recordings） - * @param cityCode 地市编码（如 340100） - * @param date 日期 - * @param uuid 设备UUID - * @param fileName 文件名 - * @return 完整路径 - */ - public static String buildLocalPath(String basePath, String cityCode, Date date, String uuid, String fileName) { - String dateDir = new SimpleDateFormat(DATE_FORMAT).format(date); - return basePath + File.separator + cityCode + File.separator + dateDir + File.separator + uuid + File.separator + fileName; - } - - /** - * 从文件路径中提取文件名 - * - * @param filePath 文件路径（如 /record/2024/01/01/test.wav） - * @return 文件名（如 test.wav） - */ - public static String extractFileName(String filePath) { - if (filePath == null || filePath.isEmpty()) { - return ""; - } - int lastSlash = filePath.lastIndexOf("/"); - return lastSlash >= 0 ? filePath.substring(lastSlash + 1) : filePath; - } - /** * 生成OSS存储路径（按地市分文件夹） * @@ -62,36 +26,4 @@ public class FilePathUtil { String dateDir = new SimpleDateFormat(DATE_FORMAT).format(date); return cityCode + "/" + dateDir + "/" + fileName; } - - /** - * 生成OSS存储的日期目录（已废弃，请使用 buildOssPath） - * - * @param date 日期 - * @return 日期目录（如 20240101） - */ - @Deprecated - public static String formatDateDir(Date date) { - return new SimpleDateFormat(DATE_FORMAT).format(date); - } - - /** - * 构建同步标记文件路径 - * - * @param basePath 基础路径 - * @param deviceId 设备ID - * @return 标记文件路径 - */ - public static String buildSyncMarkerPath(String basePath, String deviceId) { - return basePath + File.separator + ".sync-marker" + File.separator + deviceId + ".time"; - } - - /** - * 构建同步标记目录路径 - * - * @param basePath 基础路径 - * @return 标记目录路径 - */ - public static String buildSyncMarkerDir(String basePath) { - return basePath + File.separator + ".sync-marker"; - } } diff --git a/src/main/java/com/threecloud/dataserviceyy/util/FileUploadUtil.java b/src/main/java/com/threecloud/dataserviceyy/util/FileUploadUtil.java index 1663293..c51ce5b 100644 --- a/src/main/java/com/threecloud/dataserviceyy/util/FileUploadUtil.java +++ b/src/main/java/com/threecloud/dataserviceyy/util/FileUploadUtil.java @@ -13,114 +13,49 @@ import org.springframework.util.MultiValueMap; import org.springframework.web.client.RestTemplate; import javax.annotation.PostConstruct; -import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.*; import java.io.IOException; import java.net.HttpURLConnection; +import java.security.cert.X509Certificate; /** * 文件上传工具类 * 负责上传录音文件到 OSS，并返回完整的访问 URL - * - * 【配置说明】 - * vaa-sync.oss.base-url: OSS服务基础地址，用于拼接完整URL - * vaa-sync.oss.upload-url: OSS上传接口地址 - * vaa-sync.oss.appcode/appid/appsecret: OSS认证信息 + * + * 【接口】POST /oss/upload (multipart: file + path) + * 【响应】{fileName, fileUrl, fileKey, fileSize, fileType} */ @Component public class FileUploadUtil { private static final Logger logger = LoggerFactory.getLogger(FileUploadUtil.class); - // OSS配置（从配置文件读取） - @Value("${ftp-sync.oss.base-url:http://53.1.194.59:9090}") + @Value("${ftp-sync.oss.base-url:http://53.1.211.7/apiOss}") private String ossBaseUrl; - @Value("${ftp-sync.oss.upload-url:http://53.1.194.59:9090/apiOss/oss/fileUpload}") - private String ossUploadUrl; - - @Value("${ftp-sync.oss.appcode:dataservice-yy}") - private String ossAppcode; - - @Value("${ftp-sync.oss.appid:371a3368-e28e-4ba3-95a3-c31c19cf0ad0}") - private String ossAppid; - - @Value("${ftp-sync.oss.appsecret:06a6a80e-f9d2-4b3b-acc0-8d182c876074}") - private String ossAppsecret; - private RestTemplate restTemplate; @PostConstruct public void init() { restTemplate = createRestTemplate(); - logger.info("文件上传工具初始化完成"); - logger.info(" OSS基础地址: {}", ossBaseUrl); - logger.info(" 上传接口: {}", ossUploadUrl); + logger.info("文件上传工具初始化完成, OSS基础地址: {}", ossBaseUrl); } /** - * 上传WAV录音文件到OSS - * - * 存储路径格式: voice/{cityName}/{cityCode}/{date}/{filename} - * 示例: voice/合肥/340100/20240101/IN-xxx.wav - * - * 返回完整URL格式: {ossBaseUrl}/voice/{cityName}/{cityCode}/{date}/{filename} - * 示例: http://53.1.194.59:9090/voice/合肥/340100/20240101/IN-xxx.wav + * 上传录音文件到OSS + * + * 存储路径格式: voice/{ossPath} + * 示例: voice/340400/20240101/12345_67890.mp3 * - * @param cityName 地市名称（如 合肥） * @param ossPath OSS存储路径（格式: cityCode/date/filename） - * @param wavFileName WAV文件名 - * @param wavData 文件字节数组 + * @param fileName 文件名 + * @param fileData 文件字节数组 * @return 完整的文件访问URL */ - public String uploadWav(String cityName, String ossPath, String wavFileName, byte[] wavData) { - // 构建OSS对象名: voice/{cityName}/{cityCode}/{date}/{filename} - String objectName = "voice/" + cityName + "/" + ossPath; - logger.debug("构建OSS对象名: {}", objectName); - - // 上传文件 - String relativeUrl = uploadFile(objectName, wavData); - - // 返回完整URL - String fullUrl = buildFullUrl(relativeUrl); - logger.info("文件上传完成，完整URL: {}", fullUrl); - - return fullUrl; - } + public String uploadWav(String ossPath, String fileName, byte[] fileData) { + String path = "voice/" + ossPath; + logger.debug("上传文件到OSS, path={}, 大小={} bytes", path, fileData.length); - /** - * 构建完整的文件访问URL - * - * 如果返回的是相对路径，拼接 base-url - * 如果返回的已经是完整URL，直接返回 - * - * @param url 从OSS返回的URL（可能是相对路径或完整URL） - * @return 完整的访问URL - */ - private String buildFullUrl(String url) { - if (url == null || url.isEmpty()) { - return ""; - } - - // 如果已经是完整URL，直接返回 - if (url.startsWith("http://") || url.startsWith("https://")) { - return url; - } - - // 拼接基础URL - String baseUrl = ossBaseUrl.endsWith("/") ? ossBaseUrl.substring(0, ossBaseUrl.length() - 1) : ossBaseUrl; - String relativePath = url.startsWith("/") ? url : "/" + url; - - return baseUrl + relativePath; - } - - /** - * 上传文件到OSS - * - * @param fileName 文件在OSS中的路径（如 voice/合肥/340100/...） - * @param fileData 文件字节数组 - * @return OSS返回的文件路径或URL - */ - private String uploadFile(String fileName, byte[] fileData) { try { HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.MULTIPART_FORM_DATA); @@ -133,16 +68,13 @@ public class FileUploadUtil { }; MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("files", byteResource); - body.set("appcode", ossAppcode); - body.set("appid", ossAppid); - body.set("appsecret", ossAppsecret); + body.add("file", byteResource); + body.add("path", path); HttpEntity> requestEntity = new HttpEntity<>(body, headers); - logger.debug("开始上传文件到OSS: {}, 大小: {} bytes", fileName, fileData.length); ResponseEntity response = restTemplate.exchange( - ossUploadUrl, + ossBaseUrl + "/oss/upload", HttpMethod.POST, requestEntity, ResultEntity.class @@ -150,17 +82,18 @@ public class FileUploadUtil { if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { ResultEntity result = response.getBody(); - logger.debug("OSS返回结果: {}", result.toString()); if (result.getCode() == ResultEntity.StatusCode.SUCCESS.getCode()) { - String url = parseUrlFromContent(result.getContent()); - logger.debug("OSS返回URL: {}", url); - return url; + String fileUrl = parseFileUrl(result.getContent()); + logger.info("文件上传完成, URL: {}", fileUrl); + return fileUrl; } else { throw new RuntimeException("上传失败, code=" + result.getCode() + ", msg=" + result.getMsg()); } } else { throw new RuntimeException("上传失败, HTTP状态码: " + response.getStatusCode()); } + } catch (RuntimeException e) { + throw e; } catch (Exception e) { logger.error("文件上传异常: {}", fileName, e); throw new RuntimeException("文件上传失败: " + e.getMessage(), e); @@ -168,124 +101,90 @@ public class FileUploadUtil { } /** - * 从 ResultEntity.content 中解析文件URL - * - * 支持多种返回格式： - * 1. Map 对象: {"url": "http://xxx", "fileName": "xxx.wav"} - * 2. JSON 字符串: "{\"url\":\"http://xxx\"}" - * 3. 纯 URL 字符串: "http://xxx/xxx.wav" - * 4. 数组格式: [{"url": "http://xxx"}] - * 5. 相对路径: "voice/xxx/xxx.wav" - * - * @param content OSS 返回的内容 - * @return 解析后的文件路径或URL + * 从响应 content 中解析 fileUrl + * + * 支持格式： + * 1. Map: {"fileUrl": "http://xxx", "fileName": "xxx.mp3"} + * 2. JSON字符串: "{\"fileUrl\":\"http://xxx\"}" + * 3. 直接URL字符串 */ - private String parseUrlFromContent(Object content) { + private String parseFileUrl(Object content) { if (content == null) { - logger.warn("OSS 返回 content 为空"); return ""; } - - // 情况1：Map 对象 + if (content instanceof java.util.Map) { java.util.Map map = (java.util.Map) content; - Object url = map.get("url"); + Object url = map.get("fileUrl"); if (url != null) { - logger.debug("从 Map 解析到 URL: {}", url); return url.toString(); } - // 尝试其他可能的 key - Object fileUrl = map.get("fileUrl"); - if (fileUrl != null) { - logger.debug("从 Map 解析到 fileUrl: {}", fileUrl); - return fileUrl.toString(); - } - Object path = map.get("path"); - if (path != null) { - logger.debug("从 Map 解析到 path: {}", path); - return path.toString(); - } - } - - // 情况2：List/Array 对象（批量上传返回） - if (content instanceof java.util.List) { - java.util.List list = (java.util.List) content; - if (!list.isEmpty()) { - Object first = list.get(0); - if (first instanceof java.util.Map) { - Object url = ((java.util.Map) first).get("url"); - if (url != null) { - logger.debug("从 List 第一个元素解析到 URL: {}", url); - return url.toString(); - } - } + url = map.get("url"); + if (url != null) { + return url.toString(); } } - - // 情况3：JSON 字符串 + String contentStr = content.toString().trim(); if (contentStr.startsWith("{") || contentStr.startsWith("[")) { try { - // 尝试解析为 JSON Object parsed = com.alibaba.fastjson2.JSON.parse(contentStr); if (parsed instanceof java.util.Map) { - return parseUrlFromContent(parsed); // 递归解析 - } - if (parsed instanceof java.util.List) { - return parseUrlFromContent(parsed); // 递归解析 + return parseFileUrl(parsed); } } catch (Exception e) { - logger.debug("解析 JSON 失败: {}", e.getMessage()); + logger.debug("解析JSON失败: {}", e.getMessage()); } } - - // 情况4：直接是 URL 字符串（以 http:// 或 https:// 开头） + if (contentStr.startsWith("http://") || contentStr.startsWith("https://")) { - logger.debug("直接返回 URL 字符串: {}", contentStr); - return contentStr; - } - - // 情况5：相对路径（以 voice/ 开头） - if (contentStr.startsWith("voice/")) { - logger.debug("返回相对路径: {}", contentStr); return contentStr; } - - // 情况6：尝试从字符串中提取 URL - if (contentStr.contains("\"url\"")) { - try { - int start = contentStr.indexOf("\"url\":\"") + 7; - int end = contentStr.indexOf("\"", start); - if (start > 6 && end > start) { - String url = contentStr.substring(start, end); - logger.debug("从字符串提取 URL: {}", url); - return url; - } - } catch (Exception e) { - logger.warn("从字符串提取 URL 失败: {}", e.getMessage()); - } + + // 相对路径，拼接 base-url + if (contentStr.startsWith("/")) { + String base = ossBaseUrl.endsWith("/") ? ossBaseUrl.substring(0, ossBaseUrl.length() - 1) : ossBaseUrl; + return base + contentStr; } - - // 兜底：直接返回字符串 - logger.warn("无法解析 URL，直接返回 content: {}", contentStr); + return contentStr; } /** - * 创建支持自定义SSL配置的RestTemplate - * 针对HTTPS请求，配置信任所有证书并关闭Hostname验证，以兼容自签名证书 + * 创建支持跳过SSL验证的RestTemplate */ private RestTemplate createRestTemplate() { - SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory() { - @Override - protected void prepareConnection(HttpURLConnection connection, String httpMethod) throws IOException { - if (ossUploadUrl.startsWith("https") && connection instanceof HttpsURLConnection) { - // HTTPS 特殊配置（如需要） + try { + TrustManager[] trustAll = new TrustManager[]{ + new X509TrustManager() { + @Override + public X509Certificate[] getAcceptedIssuers() { return null; } + @Override + public void checkClientTrusted(X509Certificate[] certs, String authType) {} + @Override + public void checkServerTrusted(X509Certificate[] certs, String authType) {} + } + }; + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustAll, new java.security.SecureRandom()); + + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory() { + @Override + protected void prepareConnection(HttpURLConnection connection, String httpMethod) throws IOException { + if (connection instanceof HttpsURLConnection) { + ((HttpsURLConnection) connection).setSSLSocketFactory(sslContext.getSocketFactory()); + ((HttpsURLConnection) connection).setHostnameVerifier((hostname, session) -> true); + } + super.prepareConnection(connection, httpMethod); } - super.prepareConnection(connection, httpMethod); - } - }; - requestFactory.setBufferRequestBody(false); - return new RestTemplate(requestFactory); + }; + factory.setBufferRequestBody(false); + factory.setConnectTimeout(15000); + factory.setReadTimeout(300000); + return new RestTemplate(factory); + } catch (Exception e) { + logger.error("创建SSL RestTemplate失败，使用默认配置", e); + return new RestTemplate(); + } } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 30d11c9..68a912c 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -10,14 +10,11 @@ mybatis: type-aliases-package: com.threecloud.dataserviceyy configuration: map-underscore-to-camel-case: true - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl logging: level: root: INFO com.threecloud.dataserviceyy: DEBUG - com.threecloud.dataserviceyy.service.VaaSyncService: DEBUG - com.threecloud.dataserviceyy.util.VaaHttpUtil: DEBUG com.threecloud.dataserviceyy.util.FileUploadUtil: INFO com.threecloud.dataserviceyy.mapper: DEBUG org.springframework: WARN @@ -29,6 +26,6 @@ logging: console: "%d{HH:mm:ss.SSS} [%thread] %-5level %logger{20} - %msg%n" logback: rollingpolicy: - max-file-size: 10MB + file-name-pattern: logs/app-%d{yyyy-MM-dd}.log max-history: 30 total-size-cap: 1GB