实例

流程为

  1. 装个抓包工具
  2. 浏览器配置代理为抓包工具
  3. 在页面上点击播放后,会加载m3u8文件
  4. 抓包工具检测到特定信息后,将该m3u8地址发送给spring boot应用
  5. springboot根据收到的m3u8信息 下载视频 解密成mp4

抓包工具-mitmproxy

mitmproxy 使用docker安装 docker主机ip 192.168.88.247

检测脚本

当检测到特定的请求时,将m3u8的信息发送给springboot应用

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
jweb_url="http://192.168.88.188:8899/zz/bbc"
search_host="playCourseDemo.com"
cat > pyscript.py <<EOF
from mitmproxy import ctx
import requests

def request(flow):
def response(flow):
  print(22)
  if "$search_host" not in str(flow.request.host):
    return
  tbody = flow.response.get_text()
  findStr = '''player.playCallBack({'''
  if( findStr in tbody):
    print("send a task")
    referStr = flow.request.headers["Referer"]
    r1 = requests.post("$jweb_url",data={"callBackStr":tbody, "referStr":referStr})
EOF

启动

1
2
3
docker run --name mp -it -p 8080:8080 -p 8081:8081 \
-v `pwd`/:/mnt/pss \
mitmproxy/mitmproxy:4.0.4 mitmweb -s /mnt/pss/pyscript.py --web-iface 0.0.0.0

发现缺少模块 requests 进入容器 安装模块 重启容器

1
2
3
4
5
6
docker exec -it mp ash
pip3 install requests
exit

docker restart mp
docker logs -f mp

此时就抓包工具就好了

本地浏览器的配置

查看界面 192.168.88.247:8081
设置浏览器的http、https代理为 192.168.88.247:8080
http://mitm.it/ 安装证书到本地 否则访问https时,会有警告
装好证书后, 重启浏览器即可

ffmpeg安装

视频下载 与解密都可以用代码完成
ts转为mp4的操作需要ffmpeg
所以需要 下载 ffmpeg 配置到系统path变量中

下载地址
https://ffmpeg.zeranoe.com/builds/win64/static/

代码下载ts文件为mp4

代码部分使用的是 spring boot + scala

解析下载的逻辑都写在scala中了
然后在controller中调用了下api进行下载
这里就不放整合的过程了

入口部分

首先是入口部分 Hello.scala

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
package fluffy.mo

import fluffy.mo.mdown.M3u8Loader

object Hello extends App {
  println("--------- begin  ------")
  val playPageUrl = "https://aa.bb.com/play/123.html";
  val m3u8Url = "https://aa.cc.com/file/123/index.m3u8";
  val savePath = "F:/m3u8media/123"

  val downloader = M3u8Loader(playPageUrl, m3u8Url)
  downloader downloadTs2Path savePath

  println("main -- done")
}

次要部分

工具类 ScUtil.scala 以及上一篇的 DecodeUtil.java

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
package fluffy.mo.util

object ScUtil {
  def parseBaseM3u8Path(url: String): String = {
    // 提取 http://aa.com/bb/zz.m3u8?t=157207 中 http://aa.com/bb/
    val b1 = url.split(".m3u8?")(0)
    val tuple = b1.splitAt(b1.lastIndexOf("/"))
    tuple._1 + "/"
  }

  def l3pad(str: String): String = { // 1 -> 001 、 12-> 012
    str.reverse.padTo(3, "0").reverse.mkString("")
  }

  def l32pad02(str: String): String = { // 6 => 06
    var n_str = str;
    if (str == null) n_str = " ";
    for (i <- str.length() until 3) {
      n_str = "0" + n_str;
    }
    n_str
  }

}

m3u8的key是个url需要单独下载
M3u8KeyLoader.scala

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
package fluffy.mo.mdown

import java.net.URI
import java.net.http.{HttpClient, HttpRequest, HttpResponse}
import java.time.Duration

import fluffy.mo.util.DecodeUtil

import scala.collection.mutable

class M3u8KeyLoader(val refer: String) extends KeyLoader {
  val client = HttpClient.newBuilder.connectTimeout(Duration.ofMillis(3000)).build

  // hexdump -v -e '16/1 "%02x"' aa.key
  override def loadKeyFromUrl(keyUrl: String): String = {
    val request = HttpRequest.newBuilder.uri(URI.create(keyUrl))
      .header("Referer", refer)
      .timeout(Duration.ofMillis(3000)).build
    val response = client.send(request, HttpResponse.BodyHandlers.ofByteArray())
    DecodeUtil.byte2HexStr(response.body())
  }
}

object M3u8KeyLoader {
  val cache = new mutable.HashMap[String, M3u8KeyLoader]()

  def loaderUsingCache(refer: String): M3u8KeyLoader = {
    var result: M3u8KeyLoader = cache.getOrElseUpdate(refer, new M3u8KeyLoader(refer));
    result
  }

  def apply(refer: String): M3u8KeyLoader = loaderUsingCache(refer)
}

public interface KeyLoader {
    public String loadKeyFromUrl(String url);
}

主体部分

解析m3u8中的key iv 以及ts地址
下载、解密 、写入本地文件夹
调用 ffmpeg 将ts文件转成mp4文件

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
package fluffy.mo.mdown

import java.io.{ByteArrayInputStream, File}
import java.net.URI
import java.net.http.HttpClient.Version
import java.net.http.{HttpClient, HttpRequest, HttpResponse}
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path}
import java.time.Duration
import java.util.concurrent.CompletionStage

import fluffy.mo.mdown.M3u8Loader.parseM3u8Content
import fluffy.mo.util.{DecodeUtil, ScUtil}

import scala.concurrent.{Await, Future}
import scala.sys.process._

class M3u8Loader(playPageUrl: String, m3u8Url: String) {
  val httpClient = HttpClient.newBuilder
    .version(Version.HTTP_2).connectTimeout(Duration.ofSeconds(3))
    .build
  val (key,  iv, tsList, reqBuiler) = {
    val buildReq = (url: String) => {
      // 该方法用来给所有的http请求添加权限信息
      val builder = HttpRequest.newBuilder
      for (op <- Array(("Refer", playPageUrl))) {
        builder.header(op._1, op._2)
      }
      builder.uri(URI.create(url))
        .timeout(Duration.ofSeconds(5))
        .build
    }

    // 获取m3u8的内容字符串
    val keyLoader = M3u8KeyLoader(playPageUrl)
    val request = buildReq(m3u8Url)
    val response = httpClient.send(request, HttpResponse.BodyHandlers.ofLines())

    // 将 m3u8-str 解析成 key iv 以及ts-uri
    val (key, iv, tsUrlList) = parseM3u8Content(response.body().toArray.map(_.toString), keyLoader)
    val tsBasePath = m3u8Url.splitAt(m3u8Url.lastIndexOf("/") + 1)._1
    // ts-uri 转为 ts-url
    val fullUrl = tsUrlList.map(tsBasePath + _)
    // (key: String,  iv: String, tsList: Seq[String], reqBuiler: (String) => HttpRequest)
    (key, iv, fullUrl, buildReq)
  }

  def downloadTs2Path(dictPath: String): Unit = {
    Files.createDirectory(Path.of(dictPath))

    asyncDownload (dictPath)
    convert2mp4 (dictPath)
    // todo 在生成MP4文件后 删除 ts文件 和 vlist.txt 文件
  }

  def syncDownload(dictPath: String): Unit = {
    println(s"待下载ts文件数 -- ${tsList.size}")
    val sb = new StringBuilder("") // 统计下载清单

    for ((url, index) <- tsList.zipWithIndex) {
      val request = reqBuiler(url)
      val fileName = s"${ScUtil.l3pad(index.toString)}.ts";
      sb.append(s"file '${fileName}'\n")

      val response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray())
      val bytes = response.body()
      val decodeBytes = DecodeUtil.decode(bytes, key, iv)
      Files.copy(new ByteArrayInputStream(decodeBytes), Path.of(dictPath, fileName));
    }
    Files.writeString(Path.of(dictPath, "vlist.txt"), sb.toString(), StandardCharsets.UTF_8)
  }

  import scala.compat.java8.FutureConverters.toScala
  import scala.concurrent.ExecutionContext.Implicits.global

  def asyncDownload(dictPath: String): Unit = {
    val tsSize = tsList.size
    println(s"待下载ts文件数--${tsSize}")
    val sb = new StringBuilder("") // 统计下载清单

    val tsFutures = tsList.zipWithIndex.map(x => {
      // 设置下载参数
      val (url, index) = x
      val request = reqBuiler(url)
      val fileName = s"${ScUtil.l3pad(index.toString)}.ts";
      sb.append(s"file '${fileName}'\n")
      (request, fileName)
    }) // 接下来启动并行下载
      .par.map(x => {
      val (request, fileName) = x
      val jfuture = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray())
        .asInstanceOf[CompletionStage[HttpResponse[Array[Byte]]]]
      val scafuture = toScala[HttpResponse[Array[Byte]]](jfuture)
      val taskCount = scafuture.map(x => {
        val bytes = x.body()
        // 下载完成后, 解码写入本地文件
        val decodeBytes = DecodeUtil.decode(bytes, key, iv)
        Files.copy(new ByteArrayInputStream(decodeBytes), Path.of(dictPath, fileName));
        1
      })
      taskCount
    })
    val taskCount = Future.sequence(tsFutures.toArray.toSeq)
    val costSeconds = tsSize * 8 // 每个ts处理8秒(下载时间 + 解密时间)
    val result = Await.result(taskCount, scala.concurrent.duration.Duration.create(costSeconds, "s"))

    println(s"共下载ts文件个数 == ${result.sum}")
    // ts文件清单
    Files.writeString(Path.of(dictPath, "vlist.txt"), sb.toString(), StandardCharsets.UTF_8)
  }

  def convert2mp4(dictPath: String): Unit = {
    val cmd = s"ffmpeg -f concat -i ${dictPath}/vlist.txt -c copy ${dictPath}/output.mp4"
    cmd !!
  }

}

object M3u8Loader {
  def apply(playPageUrl: String, m3u8Url: String): M3u8Loader = new M3u8Loader(playPageUrl, m3u8Url)

  def parseM3u8Content(lines: Array[String], keyLoader: KeyLoader) = {
    val tuple = lines.span(!_.startsWith("#EXTINF:"))
    val header = tuple._1.toSeq
    val body = tuple._2.toSeq
    // ts-list
    val tsUrlList = for (elem <- body if !elem.startsWith("#EXT") if elem.trim.length > 0) yield elem

    val aesLine = header(header.indexWhere(_.startsWith("#EXT-X-KEY:")))
    val keyAndIvRex = """.*="(.*?)",IV=0x([0-9a-fA-F]{32})""".r
    // iv
    val keyAndIvRex(keyUrl, iv) = aesLine

    val key = keyLoader.loadKeyFromUrl(keyUrl)
    (key, iv, tsUrlList)
  }
}

下载ts时,起先采用的串行阻塞逻辑
后来改为并行下载、解析写入本地
最后阻塞式获取结果