// // BFRecordExport.swift // BFRecordScreenKit // // Created by 胡志强 on 2021/11/25. // 录屏视频导出 import AVFoundation import BFCommonKit import BFMediaKit import Foundation import GPUImage import Photos import UIKit public enum ExportError: Int { case FileNotExist = -31001 case DataLost = -31002 case VoiceLost = -31003 case TotalDurError = -31004 case ExportExcept = -31005 case DiskNoSpace = -31006 } public class BFRecordExport { public var progress: ((Float) -> Void)? public var exportCompletion: ((Error?, URL?) -> Void)? public var originSoundVolumn: Float = 1.0 // 无录音时原声大小 public var originSoundInRecordVolumn: Float = 0.0 // 录音时原声大小 var titleStickers = [PQEditSubTitleModel]() var voiceList = [PQEditVisionTrackMaterialsModel]() var videoStickers = [PQEditVisionTrackMaterialsModel]() public var data: [BFRecordItemModel]? { didSet { if data?.count ?? 0 > 0 { for item in data! { item.generationTimeRanges() } } } } var count = 0 var stickerRanges = [CMTimeRange]() var exporter: PQCompositionExporter? // var mStickers = [PQEditVisionTrackMaterialsModel]() deinit {} public init() {} // MARK: - /// synthesisAll: 合成所有还是只合成录音部分 public func startExprot(synthesisAll: Bool) { // 1,背景视频素材 if let itemModels = data { var totalDur = 0.0 titleStickers.removeAll() voiceList.removeAll() videoStickers.removeAll() // 切割视频素材 for (_, itemModel) in itemModels.enumerated() { // 保留录音部分时，是否按录音顺序合成最终视频； // 如果需要排序，则排视频的顺序；否则排音频的顺序 let needSort = false if needSort { } else { // 音频排序 var useVoices = itemModel.getUsedVoices() useVoices.sort { m1, m2 in m1.startCMTime.seconds < m2.startCMTime.seconds } // 字幕排序 itemModel.titleStickers.sort { model1, model2 in model1.timelineIn < model2.timelineIn } } switch itemModel.mediaType { case .Image: totalDur += parseImage(synthesisAll: synthesisAll, itemModel: itemModel, totalDur: totalDur) case .some(.Video): let dur = parseVideo(synthesisAll: synthesisAll, itemModel: itemModel, totalDur: totalDur) if dur >= 0 { totalDur += dur }else{ // -1 时出错 return } case .some(.Camera): totalDur += parseCamera(synthesisAll: synthesisAll, itemModel: itemModel, totalDur: totalDur) case .none: break } // 视频处理 } beginExport(synthesisAll: synthesisAll) } } func parseImage(synthesisAll: Bool, itemModel: BFRecordItemModel, totalDur:Double) -> Double{ if !synthesisAll, itemModel.getUsedVoices().count == 0 { // 图片无录音在保留模式里不合成 return 0 } var duration = itemModel.materialDuraion.seconds if itemModel.getUsedVoices().count == 0 { // 图片无录音保持2s duration = 2 let voice = PQEditVisionTrackMaterialsModel() voice.model_in = 0.0 voice.out = 2.0 voice.aptDuration = 2.0 // voice.voiceType = VOICETYPT.None.rawValue voice.volumeGain = 100 voiceList.append(voice) } else { // for mod in itemModel.getUsedVoices() { let sticker = PQEditVisionTrackMaterialsModel() sticker.model_in = 0 sticker.out = mod.endCMTime.seconds - mod.startCMTime.seconds sticker.timelineIn = totalDur + mod.startCMTime.seconds sticker.timelineOut = totalDur + mod.endCMTime.seconds sticker.aptDuration = sticker.out sticker.duration = sticker.out sticker.locationPath = mod.wavFilePath sticker.volumeGain = 100 // Float64(model.volume) voiceList.append(sticker) } } let sticker = splitBaseMaterial(timelineIn: totalDur, model_in: 0, duration: duration) sticker.originalUImage = itemModel.coverImg sticker.volumeGain = 0 sticker.type = StickerType.IMAGE.rawValue videoStickers.append(sticker) BFLog(1, message: "image sticker - timIn:\(sticker.timelineIn), modIn:\(sticker.model_in), dur:\(duration)") for titleS in itemModel.titleStickers { // let leng = titleS.timelineOut - titleS.timelineIn let newTitleSticker = PQEditSubTitleModel() titleStickers.append(newTitleSticker) newTitleSticker.text = titleS.text newTitleSticker.setting = titleS.setting newTitleSticker.timelineIn = titleS.timelineIn + CMTime(seconds: totalDur, preferredTimescale: 1000) newTitleSticker.timelineOut = titleS.timelineOut + CMTime(seconds: totalDur, preferredTimescale: 1000) } return duration } func parseVideo(synthesisAll: Bool, itemModel: BFRecordItemModel, totalDur:Double) -> Double { var subDur = 0.0 if let localPath = itemModel.localPath { if !FileManager.default.fileExists(atPath: localPath) { let error = NSError(domain: "err", code: ExportError.FileNotExist.rawValue, userInfo: ["msg": "file not exist"]) exportCompletion?(error as Error, nil) return subDur } if synthesisAll { let drangs = itemModel.dealedDurationRanges for srange in drangs { let range = srange.range let sticker = splitBaseMaterial(timelineIn: totalDur + subDur, model_in: range.start.seconds, duration: range.duration.seconds) sticker.locationPath = localPath sticker.volumeGain = Float64(srange.isRecord ? originSoundInRecordVolumn * 100 : originSoundVolumn * 100) videoStickers.append(sticker) subDur += range.duration.seconds if srange.isRecord { itemModel.getUsedVoices().forEach { mod in let sticker = PQEditVisionTrackMaterialsModel() sticker.model_in = 0 sticker.out = mod.endCMTime.seconds - mod.startCMTime.seconds sticker.timelineIn = totalDur + mod.startCMTime.seconds sticker.timelineOut = totalDur + mod.endCMTime.seconds sticker.aptDuration = sticker.out sticker.duration = sticker.out sticker.locationPath = mod.wavFilePath sticker.volumeGain = 100 // Float64(model.volume) voiceList.append(sticker) let ttsAudioAsset = AVURLAsset(url: URL(fileURLWithPath: sticker.locationPath), options: avAssertOptions) BFLog(message: "导出使用的音频文件 timelineIn:\(sticker.timelineIn) timelineOut\(sticker.timelineOut) sticker.locationPath\( sticker.locationPath) tss语音时长\(ttsAudioAsset.duration.seconds)") } } } for titleS in itemModel.titleStickers { let newTitleSticker = PQEditSubTitleModel() titleStickers.append(newTitleSticker) newTitleSticker.text = titleS.text newTitleSticker.setting = titleS.setting newTitleSticker.timelineIn = titleS.timelineIn + CMTime(seconds: totalDur, preferredTimescale: 1000) newTitleSticker.timelineOut = titleS.timelineOut + CMTime(seconds: totalDur, preferredTimescale: 1000) } } else { // 只保留录音部分 var drangs = itemModel.dealedDurationRanges.filter { srange in srange.isRecord == true } let needSort = false if needSort { drangs.sort { range1, range2 in range1.index < range2.index } } for (index, srange) in drangs.enumerated() { let range = srange.range let sticker = splitBaseMaterial(timelineIn: totalDur + subDur, model_in: range.start.seconds, duration: range.duration.seconds) sticker.locationPath = localPath sticker.volumeGain = Float64(srange.isRecord ? originSoundInRecordVolumn * 100 : originSoundVolumn * 100) videoStickers.append(sticker) let voiceSticker = itemModel.voiceStickers[index] // mdf by ak if(itemModel.voiceChangeStickers.count > 0){//有变音 let voiceChangeModels = itemModel.voiceChangeStickers.filter { mod in mod.recordId == voiceSticker.recordId } for voice in voiceChangeModels{ BFLog(2, message: "当前变音所属:\(voiceSticker.recordId ?? "") text:\(voice.wavFilePath ?? "")") let voiceChangeSt = PQEditVisionTrackMaterialsModel() voiceChangeSt.model_in = 0 voiceChangeSt.out = voice.endCMTime.seconds - voice.startCMTime.seconds voiceChangeSt.timelineIn = totalDur + subDur + (voice.startCMTime.seconds - voiceSticker.startCMTime.seconds) voiceChangeSt.timelineOut = voiceChangeSt.timelineIn + voiceChangeSt.out voiceChangeSt.aptDuration = voiceChangeSt.out voiceChangeSt.duration = voiceChangeSt.out voiceChangeSt.locationPath = voice.wavFilePath voiceChangeSt.volumeGain = 100 // Float64(model.volume) voiceList.append(voiceChangeSt) } }else{//无变音 let voice = PQEditVisionTrackMaterialsModel() voice.model_in = 0 voice.out = voiceSticker.endCMTime.seconds - voiceSticker.startCMTime.seconds voice.timelineIn = totalDur + subDur voice.timelineOut = totalDur + subDur + voice.out voice.aptDuration = voice.out voice.duration = voice.out voice.locationPath = voiceSticker.wavFilePath voice.volumeGain = 100 // Float64(model.volume) voiceList.append(voice) } let titleModels = itemModel.titleStickers.filter { mod in mod.recordId == voiceSticker.recordId } // 字幕的时间点是以相关录音的原点为坐标计算的。 for titleS in titleModels { let newTitleSticker = PQEditSubTitleModel() titleStickers.append(newTitleSticker) newTitleSticker.text = titleS.text newTitleSticker.setting = titleS.setting newTitleSticker.timelineIn = titleS.timelineIn + CMTime(seconds: totalDur + subDur , preferredTimescale: 1000) - voiceSticker.startCMTime newTitleSticker.timelineOut = titleS.timelineOut + CMTime(seconds: totalDur + subDur, preferredTimescale: 1000) - voiceSticker.startCMTime BFLog(1, message: "timein - \(newTitleSticker.timelineIn)， out - \(newTitleSticker.timelineOut)") } subDur += range.duration.seconds } } } return subDur } func parseCamera(synthesisAll: Bool, itemModel: BFRecordItemModel, totalDur:Double) -> Double { var duration = itemModel.materialDuraion.seconds // 视频分解 for mod in itemModel.videoStickers { let sticker = PQEditVisionTrackMaterialsModel() sticker.type = StickerType.VIDEO.rawValue sticker.canvasFillType = stickerContentMode.aspectFitStr.rawValue sticker.model_in = 0 sticker.out = (mod.timelineCMOut - mod.timelineCMIn).seconds sticker.timelineIn = totalDur + mod.timelineCMIn.seconds sticker.timelineOut = totalDur + mod.timelineCMOut.seconds sticker.aptDuration = sticker.out sticker.duration = sticker.out sticker.locationPath = mod.locationPath.replacingOccurrences(of: documensDirectory, with: "") sticker.volumeGain = 1 // Float64(model.volume) videoStickers.append(sticker) } // 音频分解 for mod in itemModel.getUsedVoices() { let sticker = PQEditVisionTrackMaterialsModel() sticker.model_in = 0 sticker.type = StickerType.VOICE.rawValue sticker.out = mod.endCMTime.seconds - mod.startCMTime.seconds sticker.timelineIn = totalDur + mod.startCMTime.seconds sticker.timelineOut = totalDur + mod.endCMTime.seconds sticker.aptDuration = sticker.out sticker.duration = sticker.out sticker.locationPath = mod.wavFilePath sticker.volumeGain = 100 // Float64(model.volume) voiceList.append(sticker) } // 字幕分解 for titleS in itemModel.titleStickers { let newTitleSticker = PQEditSubTitleModel() titleStickers.append(newTitleSticker) newTitleSticker.text = titleS.text newTitleSticker.setting = titleS.setting newTitleSticker.timelineIn = titleS.timelineIn + CMTime(seconds: totalDur, preferredTimescale: 1000) newTitleSticker.timelineOut = titleS.timelineOut + CMTime(seconds: totalDur, preferredTimescale: 1000) } return duration } public func cancelExport() { exporter?.cancel() } public func clearFileCache() { data?.forEach { itemModel in itemModel.getUsedVoices().forEach { model in if let localPath = model.wavFilePath { try? FileManager.default.removeItem(atPath: localPath) } } } } enum DispatchError: Error { case timeout } func getOutputFilePath() -> URL { var outPutMP4Path = exportVideosDirectory if !directoryIsExists(dicPath: outPutMP4Path) { createDirectory(path: outPutMP4Path) } outPutMP4Path.append("video_\(String.qe.timestamp()).mp4") return URL(fileURLWithPath: outPutMP4Path) } // 切割素材 func splitBaseMaterial(timelineIn: Double, model_in: Double, duration: Double) -> PQEditVisionTrackMaterialsModel { let bgMovieInfo: PQEditVisionTrackMaterialsModel = PQEditVisionTrackMaterialsModel() bgMovieInfo.type = StickerType.VIDEO.rawValue bgMovieInfo.timelineIn = timelineIn bgMovieInfo.timelineOut = timelineIn + duration bgMovieInfo.model_in = model_in bgMovieInfo.out = model_in + duration bgMovieInfo.canvasFillType = stickerContentMode.aspectFitStr.rawValue bgMovieInfo.volumeGain = 1 bgMovieInfo.aptDuration = bgMovieInfo.timelineOut bgMovieInfo.duration = bgMovieInfo.timelineOut BFLog(1, message: "hhh- timIn:\(timelineIn), modIn:\(model_in), dur:\(duration)") return bgMovieInfo } // 因为titleStickers 是传递过来的，会修改timelinein，需要重新生成，以免影响原来的数据 // voiceList是考虑到图片有时候没有录音，在保留全部时，需要添加一个2秒的空sticker func beginExport(synthesisAll: Bool) { // 输出视频地址 // exprotVideo() // return; var outPutMP4Path = exportVideosDirectory if !directoryIsExists(dicPath: outPutMP4Path) { createDirectory(path: outPutMP4Path) } outPutMP4Path.append("video_\(String.qe.timestamp()).mp4") let outPutMP4URL = URL(fileURLWithPath: outPutMP4Path) BFLog(1, message: "导出视频地址 \(outPutMP4URL)") guard let itemData = data else { let error = NSError(domain: "err", code: ExportError.DataLost.rawValue, userInfo: ["msg": "compose_fail_lost".BFLocale]) exportCompletion?(error as Error, nil) return } // 处理导出 guard let voiceCount = data?.reduce(0, { partialResult, itemModell in itemModell.getUsedVoices().count + partialResult }) else { BFLog(1, message: "getUsedVoices() count += nil") let error = NSError(domain: "err", code: ExportError.VoiceLost.rawValue, userInfo: ["msg": "getUsedVoices() count += nil"]) exportCompletion?(error as Error, nil) return } guard let totalDuration = data?.reduce(0.0, { partialResult, itemModell in var modelDuraion = 0.0 if itemModell.mediaType == .Image { if itemModell.getUsedVoices().count == 0, synthesisAll { modelDuraion += 2 } else { modelDuraion = itemModell.materialDuraion.seconds } } else if itemModell.mediaType == .Video { modelDuraion = itemModell.dealedDurationRanges.reduce(0.0) { partialResult, srange in // partialResult + (!synthesisAll && srange.isRecord) ? if synthesisAll { return partialResult + srange.range.duration.seconds } else { return partialResult + (srange.isRecord ? srange.range.duration.seconds : 0) } } }else if itemModell.mediaType == .Camera { modelDuraion = itemModell.materialDuraion.seconds } return (partialResult ?? 0) + modelDuraion }) else { let error = NSError(domain: "err", code: ExportError.TotalDurError.rawValue, userInfo: ["msg": "时长计算出错"]) exportCompletion?(error as Error, nil) return } // MARK: - 声音合成 // 有录音操作或者多个视频，就会进入合成步骤，否则就是一个没有处理的素材，直接导出就行了 if voiceCount > 0 || videoStickers.count > 1 { let (audioMix, composition) = mergeAudio(videoStickers: videoStickers, audios: voiceList, synthesisAll: synthesisAll) let outputSize: CGSize = CGSize(width: 1080, height: Int(1080 * CGFloat(UIScreen.main.bounds.size.height / UIScreen.main.bounds.size.width))) BFLog(message: "输出视频大小:\(outputSize)") // add by ak 有字幕数据 & 显示字幕开关打开 添加字幕filter var subtitleFilter:PQSubTitleFilter? if titleStickers.count > 0,titleStickers.first?.setting.subtitleIsShow ?? true { subtitleFilter = PQSubTitleFilter(st: titleStickers, inputSize: outputSize) } exporter = PQCompositionExporter(asset: composition, videoComposition: nil, audioMix: audioMix, filters:nil,stickers: videoStickers, animationTool: nil, exportURL: outPutMP4URL,subTitleFilter:subtitleFilter) var orgeBitRate = Int(outputSize.width * outputSize.height * 3) for stick in videoStickers { if stick.type == StickerType.VIDEO.rawValue, stick.locationPath.count > 0 { let asset = AVURLAsset(url: URL(fileURLWithPath: stick.locationPath), options: avAssertOptions) let cbr = asset.tracks(withMediaType: .video).first?.estimatedDataRate if Int(cbr ?? 0) > orgeBitRate { orgeBitRate = Int(cbr ?? 0) } } } BFLog(1, message: String(format: "导出设置的码率为:%.3f MB", Double(orgeBitRate) / 1024.0 / 1024.0 / 8.0)) let preSize = Double(orgeBitRate) * totalDuration / (1024 * 1024 * 8) let freeSize = PQBridgeObject.getPhoneDiskFreeSize() if preSize + 100.0 > freeSize { // 存储完后磁盘剩余至少100M let error = NSError(domain: "err", code: ExportError.DiskNoSpace.rawValue, userInfo: ["msg": "\("option_need".BFLocale)\(Int(preSize))MB,\("option_available".BFLocale)\(Int(freeSize))MB"]) exportCompletion?(error as Error, nil) return } let tempBeginExport = Date().timeIntervalSince1970 if exporter!.prepare(videoSize: outputSize, videoAverageBitRate: orgeBitRate) { exporter!.start(playeTimeRange: CMTimeRange(start: CMTime.zero, end: CMTime(seconds: totalDuration, preferredTimescale: 1000))) } exporter?.progressClosure = { [weak self] _, _, progress in // BFLog(message: "正片合成进度 \(progress * 100)%") let useProgress = progress > 1 ? 1 : progress if progress > 0 { // 更新进度 self?.progress?(useProgress) } } exporter?.completion = { [weak self] url in // 输出视频时长 if let url = url { let outSeconds = CMTimeGetSeconds(AVAsset(url: url).duration) let exportEndTime = Date().timeIntervalSince1970 BFLog(1, message: "生成视频时长为：\(outSeconds) 总用时:\(exportEndTime - tempBeginExport)") if(PQENVUtil.shared.channel == "Development"){ cShowHUB(superView: nil, msg: (outSeconds == 0) ? "compose_retry3".BFLocale : String(format: "总用时: %.2f", exportEndTime - tempBeginExport)) } self?.exportCompletion?(nil, url) } else { let error = NSError(domain: "err", code: ExportError.ExportExcept.rawValue, userInfo: ["msg": "compose_fail_export".BFLocale]) self?.exportCompletion?(error as Error, nil) cShowHUB(superView: nil, msg: "compose_fail_export".BFLocale) } // 导出完成后取消导出 self?.exporter?.cancel() } } else { // 没有处理，直接copy原文件 if let localPath = data?.first?.localPath { exportCompletion?(nil, URL(fileURLWithPath: localPath)) } } } func dealAsset() { // asset?.tracks.first(where: { track in // if track.mediaType == .audio{ // // } // }) } func getVideoSize(asset: AVURLAsset) -> CGSize { var size = CGSize.zero asset.tracks.forEach { track in if track.mediaType == .video { let realSize = __CGSizeApplyAffineTransform(track.naturalSize, track.preferredTransform) size = CGSize(width: ceil(abs(realSize.width)), height: ceil(abs(realSize.height))) } } return size } } extension BFRecordExport { func mergeAudio(videoStickers: [PQEditVisionTrackMaterialsModel], audios: [PQEditVisionTrackMaterialsModel]?, synthesisAll: Bool) -> (AVMutableAudioMix, AVMutableComposition) { let composition = AVMutableComposition() let audioMix = AVMutableAudioMix() var tempParameters = [AVMutableAudioMixInputParameters]() var totalDuration: Float64 = 0 for sticker in videoStickers { if sticker.volumeGain == 0 { // 如果添加了会有刺啦音 BFLog(message: "音频音量 为0 不添加") continue } // sticker.volumeGain = 50 totalDuration = max(totalDuration, sticker.duration) tempParameters += PQPlayerViewModel.dealWithMaterialTrack(stickerModel: sticker, composition: composition) } if let voices = audios { if synthesisAll { tempParameters += mergeRecordVoiceAll(voices: voices, composition) } else { tempParameters += mergeRecordVoiceOnly(voices: voices, composition) } } audioMix.inputParameters = tempParameters return (audioMix, composition) } func mergeRecordVoiceOnly(voices: [PQEditVisionTrackMaterialsModel], _ composition: AVMutableComposition) -> [AVMutableAudioMixInputParameters] { var tempParameters = [AVMutableAudioMixInputParameters]() var totalDur: Double = 0.0 for model in voices { if model.volumeGain == 0 { // 如果添加了会有刺啦音 BFLog(message: "音频音量 为0 不添加") continue } tempParameters += PQPlayerViewModel.dealWithMaterialTrack(stickerModel: model, composition: composition) totalDur += model.aptDuration } return tempParameters } func mergeRecordVoiceAll(voices: [PQEditVisionTrackMaterialsModel], _ composition: AVMutableComposition) -> [AVMutableAudioMixInputParameters] { var tempParameters = [AVMutableAudioMixInputParameters]() var totalDur = 0.0 for model in voices { if model.volumeGain == 0 { // 如果添加了会有刺啦音 BFLog(message: "音频音量 为0 不添加") continue } tempParameters += PQPlayerViewModel.dealWithMaterialTrack(stickerModel: model, composition: composition) } return tempParameters } }