diff --git a/build.gradle b/build.gradle index 66d17a1..e937955 100644 --- a/build.gradle +++ b/build.gradle @@ -50,6 +50,8 @@ dependencies { // https://mvnrepository.com/artifact/org.springframework/spring-webmvc implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.3.0' + + //Webflux implementation 'org.springframework.boot:spring-boot-starter-webflux' @@ -67,6 +69,12 @@ dependencies { // Google Cloud Vision implementation 'com.google.cloud:google-cloud-vision:3.61.0' + // Google cloud speech + implementation 'com.google.cloud:google-cloud-speech:4.39.0' + + //Google API + + // Logstash Logback Encoder implementation("net.logstash.logback:logstash-logback-encoder:8.0") } diff --git a/src/main/java/hongik/Todoing/domain/todo/domain/Todo.java b/src/main/java/hongik/Todoing/domain/todo/domain/Todo.java index 93218a4..c3f6385 100644 --- a/src/main/java/hongik/Todoing/domain/todo/domain/Todo.java +++ b/src/main/java/hongik/Todoing/domain/todo/domain/Todo.java @@ -40,7 +40,7 @@ public class Todo extends BaseEntity { @Column(name = "label_id", nullable = false) private Long labelId; - @Column(name = "verification_id", nullable = false) + @Column(name = "verification_id", nullable = true) private Long verification_id; @OneToMany(mappedBy = "todo", cascade = CascadeType.ALL, orphanRemoval = true) diff --git a/src/main/java/hongik/Todoing/domain/verification/controller/TodoVerificationController.java b/src/main/java/hongik/Todoing/domain/verification/controller/TodoVerificationController.java index bb0675a..f3d0ca7 100644 --- a/src/main/java/hongik/Todoing/domain/verification/controller/TodoVerificationController.java +++ b/src/main/java/hongik/Todoing/domain/verification/controller/TodoVerificationController.java @@ -6,6 +6,7 @@ import hongik.Todoing.domain.verification.domain.Verification; import hongik.Todoing.domain.verification.dto.TextAnnotationDto; import hongik.Todoing.domain.verification.dto.VerificationResponse; +import hongik.Todoing.domain.verification.service.SpeechService; import hongik.Todoing.domain.verification.service.VerificationService; import hongik.Todoing.domain.verification.service.VisionService; import hongik.Todoing.global.apiPayload.ApiResponse; @@ -25,6 +26,7 @@ public class TodoVerificationController { private final VisionService visionService; private final VerificationService verificationService; + private final SpeechService speechService; /* - 사진으로 인증하기 -> 1. 사진 업로드 -> 2. 사진에서 글자 인식 -> 3. 인식된 글자가 할 일의 인증 문구와 일치하는지 확인 @@ -56,7 +58,7 @@ public ApiResponse verifyTextTest(@RequestPart("image")MultipartFile image) t return ApiResponse.onSuccess(result); } - @Operation(summary = "사진 인증 - 라벨 감지 테스트 컨트롤러") + @Operation(summary = "[x]사진 인증 - 라벨 감지 테스트 컨트롤러") @PostMapping("/test/labels") public ApiResponse detectLabelsTest(@RequestPart("image")MultipartFile image) throws IOException { List labels = visionService.detectLabels(image); @@ -70,7 +72,7 @@ public ApiResponse detectLabelsTest(@RequestPart("image")MultipartFile image) return ApiResponse.onSuccess(labelDescriptions); } - @Operation(summary = "사진 인증, 글과 사진 어떤 것이든 사진은 한 번에 묶어서 인증이 가능합니다.") + @Operation(summary = "[x]사진 인증, 글과 사진 어떤 것이든 사진은 한 번에 묶어서 인증이 가능합니다.") @PostMapping("/image") public ApiResponse verifyTodoImage( @AuthenticationPrincipal PrincipalDetails principal, @@ -81,4 +83,40 @@ public ApiResponse verifyTodoImage( return ApiResponse.onSuccess(response); } + @Operation(summary = "음성 인증 - STT 결과 transcript를 기반으로 투두 인증") + @PostMapping("/voice") + public ApiResponse verifyTodoVoice( + @AuthenticationPrincipal PrincipalDetails principal, + @RequestParam Long todoId, + @RequestParam String transcript + ) { + VerificationResponse response = + verificationService.verifyTodoVoice(principal.getMember(), todoId, transcript); + return ApiResponse.onSuccess(response); + } + + @Operation(summary = "텍스트 인증 - 사용자가 작성한 텍스트(50자 내외)로 투두 인증") + @PostMapping("/text") + public ApiResponse verifyTodoText( + @AuthenticationPrincipal PrincipalDetails principal, + @RequestParam Long todoId, + @RequestParam String text + ) { + VerificationResponse response = + verificationService.verifyTodoText(principal.getMember(), todoId, text); + return ApiResponse.onSuccess(response); + } + + @PostMapping("/voice/file") + public ApiResponse verifyTodoVoiceFile( + @AuthenticationPrincipal PrincipalDetails principal, + @RequestParam Long todoId, + @RequestPart("audio") MultipartFile audioFile + ) throws IOException { + String transcript = speechService.stt(audioFile); // 음성 → 텍스트 + VerificationResponse response = + verificationService.verifyTodoVoice(principal.getMember(), todoId, transcript); + return ApiResponse.onSuccess(response); + } + } diff --git a/src/main/java/hongik/Todoing/domain/verification/domain/Verification.java b/src/main/java/hongik/Todoing/domain/verification/domain/Verification.java index 23d0ef5..ce96fbf 100644 --- a/src/main/java/hongik/Todoing/domain/verification/domain/Verification.java +++ b/src/main/java/hongik/Todoing/domain/verification/domain/Verification.java @@ -23,6 +23,10 @@ public class Verification { @Enumerated(EnumType.STRING) private VerificationType type; + private Boolean success; + + private double confidence; + @Column(name = "todo_id") private Long todoId; diff --git a/src/main/java/hongik/Todoing/domain/verification/domain/VerificationUsage.java b/src/main/java/hongik/Todoing/domain/verification/domain/VerificationUsage.java index 2b9c28f..fd127f9 100644 --- a/src/main/java/hongik/Todoing/domain/verification/domain/VerificationUsage.java +++ b/src/main/java/hongik/Todoing/domain/verification/domain/VerificationUsage.java @@ -22,4 +22,11 @@ public class VerificationUsage extends BaseEntity { @Column(name = "user_id") private Long userId; + + public void increase() { + if (usageCount == null) { + usageCount = 0; + } + usageCount++; + } } diff --git a/src/main/java/hongik/Todoing/domain/verification/dto/VerificationResult.java b/src/main/java/hongik/Todoing/domain/verification/dto/VerificationResult.java new file mode 100644 index 0000000..43e54bb --- /dev/null +++ b/src/main/java/hongik/Todoing/domain/verification/dto/VerificationResult.java @@ -0,0 +1,11 @@ +package hongik.Todoing.domain.verification.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class VerificationResult { + private final boolean success; + private final double confidence; +} diff --git a/src/main/java/hongik/Todoing/domain/verification/repository/VerificationUsageRepository.java b/src/main/java/hongik/Todoing/domain/verification/repository/VerificationUsageRepository.java new file mode 100644 index 0000000..7a93b9d --- /dev/null +++ b/src/main/java/hongik/Todoing/domain/verification/repository/VerificationUsageRepository.java @@ -0,0 +1,11 @@ +package hongik.Todoing.domain.verification.repository; + +import hongik.Todoing.domain.verification.domain.VerificationUsage; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; + +public interface VerificationUsageRepository extends JpaRepository { + + Optional findByUserId(Long userId); +} diff --git a/src/main/java/hongik/Todoing/domain/verification/service/SpeechService.java b/src/main/java/hongik/Todoing/domain/verification/service/SpeechService.java new file mode 100644 index 0000000..f83aa7d --- /dev/null +++ b/src/main/java/hongik/Todoing/domain/verification/service/SpeechService.java @@ -0,0 +1,54 @@ +package hongik.Todoing.domain.verification.service; + +import com.google.cloud.speech.v1.*; +import com.google.protobuf.ByteString; +import hongik.Todoing.global.common.AuditingAi.AuditingAI; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; + +@Service +@Slf4j +public class SpeechService { + // 음성을 텍스트로 변환하는 서비스 (STT) + @AuditingAI("음성을 통한 텍스트 변환 호출기") + public String stt(MultipartFile audioFile) throws IOException { + + // 1. 파일을 ByteString으로 변환 + ByteString audioBytes = ByteString.readFrom(audioFile.getInputStream()); + + // 2. Google Speech API용 Audio 객체 생성 + RecognitionAudio audio = RecognitionAudio.newBuilder() + .setContent(audioBytes) + .build(); + + // 3. 설정 (한국어 예시: ko-KR) + RecognitionConfig config = RecognitionConfig.newBuilder() + .setEncoding(RecognitionConfig.AudioEncoding.LINEAR16) // 파일 인코딩에 맞게 조정 필요 + .setLanguageCode("ko-KR") + .build(); + + // 4. 클라이언트 생성 후 요청 + try (SpeechClient speechClient = SpeechClient.create()) { + RecognizeRequest request = RecognizeRequest.newBuilder() + .setConfig(config) + .setAudio(audio) + .build(); + + RecognizeResponse response = speechClient.recognize(request); + + StringBuilder transcriptBuilder = new StringBuilder(); + + for (SpeechRecognitionResult result : response.getResultsList()) { + SpeechRecognitionAlternative alternative = result.getAlternativesList().get(0); + transcriptBuilder.append(alternative.getTranscript()).append(" "); + } + + String transcript = transcriptBuilder.toString().trim(); + log.info("STT transcript = {}", transcript); + return transcript; + } + } +} diff --git a/src/main/java/hongik/Todoing/domain/verification/service/VerificationService.java b/src/main/java/hongik/Todoing/domain/verification/service/VerificationService.java index e3999e5..34256d6 100644 --- a/src/main/java/hongik/Todoing/domain/verification/service/VerificationService.java +++ b/src/main/java/hongik/Todoing/domain/verification/service/VerificationService.java @@ -12,7 +12,10 @@ import hongik.Todoing.domain.verification.Adaptor.VerificationAdaptor; import hongik.Todoing.domain.verification.domain.Verification; import hongik.Todoing.domain.verification.domain.VerificationType; +import hongik.Todoing.domain.verification.domain.VerificationUsage; import hongik.Todoing.domain.verification.dto.VerificationResponse; +import hongik.Todoing.domain.verification.dto.VerificationResult; +import hongik.Todoing.domain.verification.repository.VerificationUsageRepository; import hongik.Todoing.domain.verification.validator.VerificationValidator; import hongik.Todoing.global.apiPayload.code.status.ErrorStatus; import hongik.Todoing.global.apiPayload.exception.GeneralException; @@ -25,6 +28,7 @@ import java.util.List; import java.util.Optional; +import java.util.function.Function; @Service @Slf4j @@ -39,68 +43,165 @@ public class VerificationService { private final TodoRepository todoRepository; private final LabelRepository labelRepository; private final PassValidator passValidator; + private final VerificationUsageRepository verificationUsageRepository; + // 인증 요청 처리 + // 1.사진 인증을 합니다. @Transactional public VerificationResponse verifyTodoImage(Member member, Long todoId, MultipartFile image) { - //Todo 조회 - Optional todoOptional = todoRepository.findByTodoId(todoId); - Todo todo = todoOptional.orElseThrow(() -> new GeneralException(ErrorStatus.TODO_NOT_FOUND)); + return processVerification( + member, + todoId, + VerificationType.PHOTO, // 기존 enum에 있다면 그대로 + true, // PASS 사용할지 여부 (AI 인증이면 true) + (todo) -> { // 여기서만 Vision + 분석 수행 + + // vision api 호출 + List textAnnotations; + List labelAnnotations; + + try { + textAnnotations = visionService.detectText(image); + labelAnnotations = visionService.detectLabels(image); + } catch (Exception e) { + throw new RuntimeException("Vision API 호출 실패", e); + } + + double confidence = calculateConfidence(todo, textAnnotations, labelAnnotations); + boolean success = confidence >= 0.5; + + return new VerificationResult(success, confidence); + } + ); + } - // 인증 가능한 투두인가 - verificationValidator.validVerification(member, todo); + // 2. 음성 인증을 합니다. + // 음성 -> 텍스트 변환을 외부에서 + // 이를 script로 받음 + @Transactional + public VerificationResponse verifyTodoVoice(Member member, Long todoId, String transcript) { + + return processVerification( + member, + todoId, + VerificationType.AUDIO, + true, // AI 인증 → PASS 사용 + (todo) -> { + // transcript 를 Vision 의 텍스트 결과처럼 하나의 EntityAnnotation 으로 감싸서 재사용 + EntityAnnotation fakeText = EntityAnnotation.newBuilder() + .setDescription(transcript == null ? "" : transcript) + .build(); + + double confidence = calculateConfidence( + todo, + List.of(fakeText), // 텍스트만 사용 + List.of() // 라벨은 비워둠 + ); + + boolean success = confidence >= 0.5; + return new VerificationResult(success, confidence); + } + ); + } - // vision api 호출(텍스트 + 라벨) - List textAnnotations; - List labelAnnotations; + // 3. 위치 인증 대신 일단은 텍스트 인증으로 하겠슴. + @Transactional + public VerificationResponse verifyTodoText(Member member, Long todoId, String userText) { + + return processVerification( + member, + todoId, + VerificationType.TEXT, + true, // 이것도 AI 인증으로 취급 (유사도 계산) + (todo) -> { + EntityAnnotation fakeText = EntityAnnotation.newBuilder() + .setDescription(userText == null ? "" : userText) + .build(); + + double confidence = calculateConfidence( + todo, + List.of(fakeText), + List.of() + ); + + boolean success = confidence >= 0.5; + return new VerificationResult(success, confidence); + } + ); + } - try { - textAnnotations = visionService.detectText(image); - labelAnnotations = visionService.detectLabels(image); - } catch (Exception e) { - throw new RuntimeException("Vision API 호출 실패", e); - } + // 공통 처리 메서드 + private VerificationResponse processVerification( + Member member, + Long todoId, + VerificationType type, + boolean usePass, + Function verificationLogic + ) { + // 1) Todo 조회 + Todo todo = todoRepository.findByTodoId(todoId) + .orElseThrow(() -> new GeneralException(ErrorStatus.TODO_NOT_FOUND)); + + // 2) 인증 가능한 투두인지 검증 + verificationValidator.validVerification(member, todo); - // 결과 분석 -> 인증 성공/실패 여부 확인 - double confidence = calculateConfidence(todo, textAnnotations, labelAnnotations); - boolean success = confidence >= 0.5; // 신뢰도 0.5 이상이면 성공으로 간주 + // 3) 실제 인증 로직 수행 + VerificationResult result = verificationLogic.apply(todo); + boolean success = result.isSuccess(); + double confidence = result.getConfidence(); - // 인증 기록 저장 + // 4) Verification 엔티티 저장 Verification verification = Verification.builder() - .type(VerificationType.PHOTO) + .type(type) + .todoId(todo.getTodoId()) + .success(success) + .confidence(confidence) .build(); - verificationAdaptor.save(verification); + verificationAdaptor.save(verification); - // Pass 차감 - if(success) { - // 사용권이 남아있는 PASS && 가장 만들어진지 오래된 PASS + // 5) PASS 차감 + 사용량 집계 (성공했을 때만) + if (usePass && success) { Pass pass = passAdaptor.findByUserId(member.getId()) .stream() .filter(p -> p.remainingCount() > 0) .min((p1, p2) -> p1.getCreatedAt().compareTo(p2.getCreatedAt())) .orElseThrow(() -> new GeneralException(ErrorStatus.PASS_NOT_AVAILABLE)); - pass.consume(member.getId(),passValidator); + pass.consume(member.getId(), passValidator); + // 인증 사용량 기록 + increaseUsage(member.getId()); } - // Todo 완료 처리 - - if(success){ + // 6) Todo 완료 처리 + if (success) { todo.updateComplete(true); } - log.info("인증 결과 | member={}, todo={}, success={}, confidence={}", - member.getId(), todo.getTodoId(), success, confidence); + log.info("인증 결과 | member={}, todo={}, type={}, success={}, confidence={}", + member.getId(), todo.getTodoId(), type, success, confidence); - VerificationResponse response = VerificationResponse.from(verification, todo, success, confidence); + // 7) Response 생성 + return VerificationResponse.from(verification, todo, success, confidence); + } - return response; + // 인증 사용량 +1 + private void increaseUsage(Long userId) { + VerificationUsage usage = verificationUsageRepository.findByUserId(userId) + .orElseGet(() -> VerificationUsage.builder() + .userId(userId) + .usageCount(0) + .build()); + usage.increase(); + verificationUsageRepository.save(usage); } + + /** * 결과 분석 로직 * 카테고리 내용 기반으로 텍스트/라벨 매칭 확률 계산 @@ -109,9 +210,11 @@ public VerificationResponse verifyTodoImage(Member member, Long todoId, Multipar private double calculateConfidence(Todo todo, List texts, List labels) { double score = 0.0; - LabelType label = labelRepository.findById(todo.getLabelId()).orElseThrow().getLabelName(); + LabelType label = labelRepository.findById(todo.getLabelId()) + .orElseThrow() + .getLabelName(); - String category = label.toString(); + String category = label.toString().toLowerCase(); String content = todo.getContent().toLowerCase(); // 텍스트 매칭 점수 계산 @@ -119,26 +222,27 @@ private double calculateConfidence(Todo todo, List texts, List String detected = text.getDescription().toLowerCase(); // 내용 일부 포함 시 +0.3 - if(detected.contains(content)) + if (detected.contains(content)) { score += 0.3; + } - // ㅋ테고리 키워드 포함 시 +0.4 - if(detected.contains(category)) + // 카테고리 키워드 포함 시 +0.4 + if (detected.contains(category)) { score += 0.4; - + } } - //라벨 분석 - for(EntityAnnotation labelText : labels) { - + // 라벨 분석 + for (EntityAnnotation labelText : labels) { String labelDescription = labelText.getDescription().toLowerCase(); - if(labelDescription.contains(category)) - score += 0.4; + if (labelDescription.contains(category)) { + score += 0.4; + } - if(content.contains(labelDescription)) + if (content.contains(labelDescription)) { score += 0.3; - + } } // 최대 1.0으로 제한 @@ -147,6 +251,7 @@ private double calculateConfidence(Todo todo, List texts, List // vision api 결과 해석 로직 부분 + // 일단 안쓰고 있음.. private boolean analyzeResult(List textAnnotations, List labelAnnotations, Todo todo) { diff --git a/src/main/java/hongik/Todoing/global/config/CorsConfig.java b/src/main/java/hongik/Todoing/global/config/CorsConfig.java index c8aa6d7..e64a982 100644 --- a/src/main/java/hongik/Todoing/global/config/CorsConfig.java +++ b/src/main/java/hongik/Todoing/global/config/CorsConfig.java @@ -21,7 +21,8 @@ public static CorsConfigurationSource apiConfigurationSource() { List allowedOrigins = List.of( "http://localhost:3000", "http://localhost:5173",// 개발용 React 주소 - "https://todooungi.netlify.app" // 실제 배포 주소 + "https://todooungi.netlify.app", // 실제 배포 주소 + "http://43.203.235.223" // 실제 배포 주소 ); configuration.setAllowedOrigins(allowedOrigins); diff --git a/src/main/java/hongik/Todoing/global/config/PaymentsProperties.java b/src/main/java/hongik/Todoing/global/config/PaymentsProperties.java index 5e06cd4..1153400 100644 --- a/src/main/java/hongik/Todoing/global/config/PaymentsProperties.java +++ b/src/main/java/hongik/Todoing/global/config/PaymentsProperties.java @@ -11,6 +11,7 @@ public class PaymentsProperties { private String cid; @Value("${kakaopay.secret_key}") private String secretKey; + private final String apiUrl = "https://open-api.kakaopay.com/online"; private final String localUrl = "http://localhost:8080"; private final String readyUrl = "/v1/payment/ready"; diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index e6b405b..3f5e24f 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -33,7 +33,7 @@ spring: jpa: hibernate: - ddl-auto: create + ddl-auto: update properties: hibernate: show_sql: true diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 3454360..fb8afc0 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,5 +1,5 @@ spring: profiles: active: - - dev + - local