1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

jweb_url="http://192.168.88.188:8899/zz/bbc"
search_host="playCourseDemo.com"
cat > pyscript.py <<EOF
from mitmproxy import ctx
import requests

def request(flow):
def response(flow):
  print(22)
  if "$search_host" not in str(flow.request.host):
    return
  tbody = flow.response.get_text()
  findStr = '''player.playCallBack({'''
  if( findStr in tbody):
    print("send a task")
    referStr = flow.request.headers["Referer"]
    r1 = requests.post("$jweb_url",data={"callBackStr":tbody, "referStr":referStr})
EOF

1
2
3

docker run --name mp -it -p 8080:8080 -p 8081:8081 \
-v `pwd`/:/mnt/pss \
mitmproxy/mitmproxy:4.0.4 mitmweb -s /mnt/pss/pyscript.py --web-iface 0.0.0.0

1
2
3
4
5
6

docker exec -it mp ash
pip3 install requests
exit

docker restart mp
docker logs -f mp

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

package fluffy.mo

import fluffy.mo.mdown.M3u8Loader

object Hello extends App {
  println("--------- begin  ------")
  val playPageUrl = "https://aa.bb.com/play/123.html";
  val m3u8Url = "https://aa.cc.com/file/123/index.m3u8";
  val savePath = "F:/m3u8media/123"

  val downloader = M3u8Loader(playPageUrl, m3u8Url)
  downloader downloadTs2Path savePath

  println("main -- done")
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

package fluffy.mo.util

object ScUtil {
  def parseBaseM3u8Path(url: String): String = {
    // 提取 http://aa.com/bb/zz.m3u8?t=157207 中 http://aa.com/bb/
    val b1 = url.split(".m3u8?")(0)
    val tuple = b1.splitAt(b1.lastIndexOf("/"))
    tuple._1 + "/"
  }

  def l3pad(str: String): String = { // 1 -> 001 、 12-> 012
    str.reverse.padTo(3, "0").reverse.mkString("")
  }

  def l32pad02(str: String): String = { // 6 => 06
    var n_str = str;
    if (str == null) n_str = " ";
    for (i <- str.length() until 3) {
      n_str = "0" + n_str;
    }
    n_str
  }

}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

package fluffy.mo.mdown

import java.net.URI
import java.net.http.{HttpClient, HttpRequest, HttpResponse}
import java.time.Duration

import fluffy.mo.util.DecodeUtil

import scala.collection.mutable

class M3u8KeyLoader(val refer: String) extends KeyLoader {
  val client = HttpClient.newBuilder.connectTimeout(Duration.ofMillis(3000)).build

  // hexdump -v -e '16/1 "%02x"' aa.key
  override def loadKeyFromUrl(keyUrl: String): String = {
    val request = HttpRequest.newBuilder.uri(URI.create(keyUrl))
      .header("Referer", refer)
      .timeout(Duration.ofMillis(3000)).build
    val response = client.send(request, HttpResponse.BodyHandlers.ofByteArray())
    DecodeUtil.byte2HexStr(response.body())
  }
}

object M3u8KeyLoader {
  val cache = new mutable.HashMap[String, M3u8KeyLoader]()

  def loaderUsingCache(refer: String): M3u8KeyLoader = {
    var result: M3u8KeyLoader = cache.getOrElseUpdate(refer, new M3u8KeyLoader(refer));
    result
  }

  def apply(refer: String): M3u8KeyLoader = loaderUsingCache(refer)
}

public interface KeyLoader {
    public String loadKeyFromUrl(String url);
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

package fluffy.mo.mdown

import java.io.{ByteArrayInputStream, File}
import java.net.URI
import java.net.http.HttpClient.Version
import java.net.http.{HttpClient, HttpRequest, HttpResponse}
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path}
import java.time.Duration
import java.util.concurrent.CompletionStage

import fluffy.mo.mdown.M3u8Loader.parseM3u8Content
import fluffy.mo.util.{DecodeUtil, ScUtil}

import scala.concurrent.{Await, Future}
import scala.sys.process._

class M3u8Loader(playPageUrl: String, m3u8Url: String) {
  val httpClient = HttpClient.newBuilder
    .version(Version.HTTP_2).connectTimeout(Duration.ofSeconds(3))
    .build
  val (key,  iv, tsList, reqBuiler) = {
    val buildReq = (url: String) => {
      // 该方法用来给所有的http请求添加权限信息
      val builder = HttpRequest.newBuilder
      for (op <- Array(("Refer", playPageUrl))) {
        builder.header(op._1, op._2)
      }
      builder.uri(URI.create(url))
        .timeout(Duration.ofSeconds(5))
        .build
    }

    // 获取m3u8的内容字符串
    val keyLoader = M3u8KeyLoader(playPageUrl)
    val request = buildReq(m3u8Url)
    val response = httpClient.send(request, HttpResponse.BodyHandlers.ofLines())

    // 将 m3u8-str 解析成 key iv 以及ts-uri
    val (key, iv, tsUrlList) = parseM3u8Content(response.body().toArray.map(_.toString), keyLoader)
    val tsBasePath = m3u8Url.splitAt(m3u8Url.lastIndexOf("/") + 1)._1
    // ts-uri 转为 ts-url
    val fullUrl = tsUrlList.map(tsBasePath + _)
    // (key: String,  iv: String, tsList: Seq[String], reqBuiler: (String) => HttpRequest)
    (key, iv, fullUrl, buildReq)
  }

  def downloadTs2Path(dictPath: String): Unit = {
    Files.createDirectory(Path.of(dictPath))

    asyncDownload (dictPath)
    convert2mp4 (dictPath)
    // todo 在生成MP4文件后 删除 ts文件 和 vlist.txt 文件
  }

  def syncDownload(dictPath: String): Unit = {
    println(s"待下载ts文件数 -- ${tsList.size}")
    val sb = new StringBuilder("") // 统计下载清单

    for ((url, index) <- tsList.zipWithIndex) {
      val request = reqBuiler(url)
      val fileName = s"${ScUtil.l3pad(index.toString)}.ts";
      sb.append(s"file '${fileName}'\n")

      val response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray())
      val bytes = response.body()
      val decodeBytes = DecodeUtil.decode(bytes, key, iv)
      Files.copy(new ByteArrayInputStream(decodeBytes), Path.of(dictPath, fileName));
    }
    Files.writeString(Path.of(dictPath, "vlist.txt"), sb.toString(), StandardCharsets.UTF_8)
  }

  import scala.compat.java8.FutureConverters.toScala
  import scala.concurrent.ExecutionContext.Implicits.global

  def asyncDownload(dictPath: String): Unit = {
    val tsSize = tsList.size
    println(s"待下载ts文件数--${tsSize}")
    val sb = new StringBuilder("") // 统计下载清单

    val tsFutures = tsList.zipWithIndex.map(x => {
      // 设置下载参数
      val (url, index) = x
      val request = reqBuiler(url)
      val fileName = s"${ScUtil.l3pad(index.toString)}.ts";
      sb.append(s"file '${fileName}'\n")
      (request, fileName)
    }) // 接下来启动并行下载
      .par.map(x => {
      val (request, fileName) = x
      val jfuture = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray())
        .asInstanceOf[CompletionStage[HttpResponse[Array[Byte]]]]
      val scafuture = toScala[HttpResponse[Array[Byte]]](jfuture)
      val taskCount = scafuture.map(x => {
        val bytes = x.body()
        // 下载完成后， 解码写入本地文件
        val decodeBytes = DecodeUtil.decode(bytes, key, iv)
        Files.copy(new ByteArrayInputStream(decodeBytes), Path.of(dictPath, fileName));
        1
      })
      taskCount
    })
    val taskCount = Future.sequence(tsFutures.toArray.toSeq)
    val costSeconds = tsSize * 8 // 每个ts处理8秒（下载时间 + 解密时间)
    val result = Await.result(taskCount, scala.concurrent.duration.Duration.create(costSeconds, "s"))

    println(s"共下载ts文件个数 == ${result.sum}")
    // ts文件清单
    Files.writeString(Path.of(dictPath, "vlist.txt"), sb.toString(), StandardCharsets.UTF_8)
  }

  def convert2mp4(dictPath: String): Unit = {
    val cmd = s"ffmpeg -f concat -i ${dictPath}/vlist.txt -c copy ${dictPath}/output.mp4"
    cmd !!
  }

}

object M3u8Loader {
  def apply(playPageUrl: String, m3u8Url: String): M3u8Loader = new M3u8Loader(playPageUrl, m3u8Url)

  def parseM3u8Content(lines: Array[String], keyLoader: KeyLoader) = {
    val tuple = lines.span(!_.startsWith("#EXTINF:"))
    val header = tuple._1.toSeq
    val body = tuple._2.toSeq
    // ts-list
    val tsUrlList = for (elem <- body if !elem.startsWith("#EXT") if elem.trim.length > 0) yield elem

    val aesLine = header(header.indexWhere(_.startsWith("#EXT-X-KEY:")))
    val keyAndIvRex = """.*="(.*?)",IV=0x([0-9a-fA-F]{32})""".r
    // iv
    val keyAndIvRex(keyUrl, iv) = aesLine

    val key = keyLoader.loadKeyFromUrl(keyUrl)
    (key, iv, tsUrlList)
  }
}