| 328 | | urisourcebin uri=$URI ! decodebin ! \ |
| 329 | | videoconvert ! video/x-raw,format=BGRA ! \ |
| 330 | | dvPre model=$MODEL ! \ |
| 331 | | dvInf model=$MODEL sock=/var/run/proxy.sock use-shm=false ! \ |
| 332 | | dvPost model=$MODEL ! \ |
| 333 | | fakesink sync=false | grep Detected |
| 334 | | }}} |
| 335 | | - the GST_PLUGIN_FEATURE_RANK is to disable the use of the v4l2jpegdec hardware decode on GStreamer 1.28 as it does not support a compatible format needed by dvPre (jet jpegdec does) |
| 336 | | * Image detection with boxing via Python |
| 337 | | - Python is incredibly useful for accessing GStreamer and handling the ARA detection frame data and imagemagick provides excellent tools for converting and drawing on images: |
| 338 | | - (optional) install lighttpd so that we can easily see our resulting images via a browser |
| 339 | | {{{#!bash |
| 340 | | apt-get install -y lighttpd |
| 341 | | # add configuration for directory listing and mapping of /root to / |
| 342 | | cat << EOF >> /etc/lighttpd/lighttpd.conf |
| 343 | | dir-listing.encoding = "utf-8" |
| 344 | | server.dir-listing = "enable" |
| 345 | | |
| 346 | | # directory access |
| 347 | | alias.url += ( |
| 348 | | "/root" => "/root", |
| 349 | | ) |
| 350 | | EOF |
| 351 | | # make the dir executable |
| 352 | | chmod ugo+x . |
| 353 | | # restart the web server |
| 354 | | /etc/init.d/lighttpd restart |
| 355 | | }}} |
| 356 | | - install imagemagick which we will use to draw named boxes for detections |
| 357 | | {{{#!bash |
| 358 | | apt-get install -y imagemagick |
| 359 | | }}} |
| 360 | | - create a dir for us to work in and create the script |
| 361 | | {{{#!bash |
| 362 | | mkdir image-detect; cd image-detect |
| 363 | | # create python script |
| 364 | | cat <<\EOF > image_detect.py |
| 365 | | #!/usr/bin/env python3 |
| 366 | | """ |
| 367 | | Ara NPU Multi-Format Universal Image Decoder |
| 368 | | ============================================ |
| 369 | | """ |
| 370 | | |
| 371 | | import ctypes |
| 372 | | import os |
| 373 | | import sys |
| 374 | | import subprocess |
| 375 | | import gi |
| 376 | | |
| 377 | | gi.require_version('Gst', '1.0') |
| 378 | | from gi.repository import Gst |
| 379 | | |
| 380 | | Gst.init(None) |
| 381 | | |
| 382 | | # Standard COCO Class Mapping for printing human-readable labels |
| 383 | | COCO_CLASSES = { |
| 384 | | 0: "person", 1: "bicycle", 2: "car", 3: "motorcycle", 4: "airplane", 5: "bus", |
| 385 | | 6: "train", 7: "truck", 8: "boat", 9: "traffic light", 10: "fire hydrant", |
| 386 | | 11: "stop sign", 12: "parking meter", 13: "bench", 14: "bird", 15: "cat", |
| 387 | | 16: "dog", 17: "horse", 18: "sheep", 19: "cow", 20: "elephant", 21: "bear", |
| 388 | | 22: "zebra", 23: "giraffe", 24: "backpack", 25: "umbrella", 26: "handbag", |
| 389 | | 27: "tie", 28: "suitcase", 29: "frisbee", 30: "skis", 31: "snowboard", |
| 390 | | 32: "sports ball", 33: "kite", 34: "baseball bat", 35: "baseball glove", |
| 391 | | 36: "skateboard", 37: "surfboard", 38: "tennis racket", 39: "bottle", |
| 392 | | 40: "wine glass", 41: "cup", 42: "fork", 43: "knife", 44: "spoon", 45: "bowl", |
| 393 | | 46: "banana", 47: "apple", 48: "sandwich", 49: "orange", 50: "broccoli", |
| 394 | | 51: "carrot", 52: "hot dog", 53: "pizza", 54: "donut", 55: "cake", |
| 395 | | 56: "chair", 57: "couch", 58: "potted plant", 59: "bed", 60: "dining table", |
| 396 | | 61: "toilet", 62: "tv", 63: "laptop", 64: "mouse", 65: "remote", 66: "keyboard", |
| 397 | | 67: "cell phone", 68: "microwave", 69: "oven", 70: "toaster", 71: "sink", |
| 398 | | 72: "refrigerator", 73: "book", 74: "clock", 75: "vase", 76: "scissors", |
| 399 | | 77: "teddy bear", 78: "hair drier", 79: "toothbrush" |
| 400 | | } |
| 401 | | |
| 402 | | class AraDetection(ctypes.Structure): |
| 403 | | _layout_ = "ms" |
| 404 | | _pack_ = 1 |
| 405 | | _fields_ = [ |
| 406 | | ("xmin", ctypes.c_float), ("ymin", ctypes.c_float), |
| 407 | | ("xmax", ctypes.c_float), ("ymax", ctypes.c_float), |
| 408 | | ("confidence", ctypes.c_float), ("class_id", ctypes.c_int32), |
| 409 | | ("class_name_ptr", ctypes.c_void_p) |
| 410 | | ] |
| 411 | | |
| 412 | | def main(): |
| 413 | | if len(sys.argv) < 3: |
| 414 | | print(f"Usage: {sys.argv[0]} <input_image> <output_image> [model]") |
| 415 | | sys.exit(1) |
| 416 | | |
| 417 | | input_image = sys.argv[1] |
| 418 | | output_image = sys.argv[2] |
| 419 | | model = "/usr/share/cnn/detection/yolov8n/model.dvm" |
| 420 | | if len(sys.argv) > 3: |
| 421 | | model = sys.argv[3] |
| 422 | | |
| 423 | | if not os.path.exists(input_image): |
| 424 | | print(f"ERROR: File '{input_image}' could not be located.") |
| 425 | | sys.exit(1) |
| 426 | | |
| 427 | | # Fetch native dimensions using ImageMagick |
| 428 | | try: |
| 429 | | dimensions = subprocess.check_output(f"identify -format '%w %h' {input_image}", shell=True).decode().split() |
| 430 | | w_native, h_native = int(dimensions[0]), int(dimensions[1]) |
| 431 | | except Exception as e: |
| 432 | | print(f"ERROR: Failed to read image properties using ImageMagick: {e}") |
| 433 | | sys.exit(1) |
| 434 | | |
| 435 | | # Print target properties cleanly |
| 436 | | print(f"\nmodel: {model}") |
| 437 | | print(f"image: {os.path.basename(input_image)} {w_native}x{h_native}") |
| 438 | | |
| 439 | | MODEL_W, MODEL_H = 640, 640 |
| 440 | | |
| 441 | | pipe_str = ( |
| 442 | | f"multifilesrc location={input_image} loop=false num-buffers=2 ! decodebin name=d ! " |
| 443 | | f"videoconvert ! videoscale ! video/x-raw,width={MODEL_W},height={MODEL_H} ! " |
| 444 | | f"videoconvert ! video/x-raw,format=BGRA ! " |
| 445 | | f"dvPre model={model} ! " |
| 446 | | f"dvInf model={model} sock=/var/run/proxy.sock use-shm=true shm-path=/dev/shm/ara_inf_ ! " |
| 447 | | f"dvPost model={model} orig-width={MODEL_W} orig-height={MODEL_H} ! " |
| 448 | | f"appsink name=mysink sync=false async=false emit-signals=true" |
| 449 | | ) |
| 450 | | |
| 451 | | # Before creating the launcher, adjust the system plugin registry ranking |
| 452 | | # so GStreamer ignores v4l2jpegdec element (as it doesn't support BGRA output) |
| 453 | | registry = Gst.Registry.get() |
| 454 | | feature = registry.lookup_feature("v4l2jpegdec") |
| 455 | | if feature: |
| 456 | | # Lower its rank to ZERO so decodebin skips over it permanently |
| 457 | | feature.set_rank(0) |
| 458 | | |
| 459 | | pipeline = Gst.parse_launch(pipe_str) |
| 460 | | sink = pipeline.get_by_name("mysink") |
| 461 | | pipeline.set_state(Gst.State.PLAYING) |
| 462 | | |
| 463 | | last_valid_raw_bytes = None |
| 464 | | |
| 465 | | while True: |
| 466 | | sample = sink.emit("pull-sample") |
| 467 | | if not sample: |
| 468 | | break |
| 469 | | buffer = sample.get_buffer() |
| 470 | | last_valid_raw_bytes = buffer.extract_dup(0, buffer.get_size()) |
| 471 | | |
| 472 | | pipeline.set_state(Gst.State.NULL) |
| 473 | | |
| 474 | | processed_detections = [] |
| 475 | | |
| 476 | | if last_valid_raw_bytes and len(last_valid_raw_bytes) >= 4: |
| 477 | | num_detections = int.from_bytes(last_valid_raw_bytes[:4], byteorder='little') |
| 478 | | |
| 479 | | if 0 < num_detections < 1000: |
| 480 | | print(f"DETECTIONS LOGGED: FOUND {num_detections} ACTIVE OBJECTS") |
| 481 | | print("-" * 70) |
| 482 | | |
| 483 | | offset = 4 |
| 484 | | ds = ctypes.sizeof(AraDetection) |
| 485 | | |
| 486 | | for i in range(num_detections): |
| 487 | | if offset + ds > len(last_valid_raw_bytes): break |
| 488 | | det = AraDetection.from_buffer_copy(last_valid_raw_bytes[offset:offset+ds]) |
| 489 | | offset += ds |
| 490 | | |
| 491 | | # Compute native image coordinate translation mapping |
| 492 | | x1_mapped = det.xmin * (w_native / MODEL_W) |
| 493 | | x2_mapped = det.xmax * (w_native / MODEL_W) |
| 494 | | y1_mapped = det.ymin * (h_native / MODEL_H) |
| 495 | | y2_mapped = det.ymax * (h_native / MODEL_H) |
| 496 | | |
| 497 | | coco_name = COCO_CLASSES.get(det.class_id, "unknown") |
| 498 | | |
| 499 | | print(f"Object {i+1}: ID={det.class_id} | Name={coco_name} | Confidence={det.confidence * 100:.1f}%") |
| 500 | | print(f" Bounding Box -> [{int(x1_mapped)}, {int(y1_mapped)}] to [{int(x2_mapped)}, {int(y2_mapped)}]") |
| 501 | | print("-" * 70) |
| 502 | | |
| 503 | | processed_detections.append((coco_name, det.confidence, x1_mapped, y1_mapped, x2_mapped, y2_mapped)) |
| 504 | | |
| 505 | | # Render final multi-object annotated canvas |
| 506 | | if processed_detections: |
| 507 | | cmd_args = [f"convert {input_image}"] |
| 508 | | for coco_name, conf, x1, y1, x2, y2 in processed_detections: |
| 509 | | ix1, iy1, ix2, iy2 = int(x1), int(y1), int(x2), int(y2) |
| 510 | | label = f"{coco_name} {conf*100:.1f}%" |
| 511 | | cmd_args.append(f'-stroke green -strokewidth 2 -fill none -draw "rectangle {ix1},{iy1} {ix2},{iy2}"') |
| 512 | | cmd_args.append(f'-stroke none -fill white -pointsize 16 -annotate +{ix1}+{iy1 - 6} "{label}"') |
| 513 | | |
| 514 | | cmd_args.append(output_image) |
| 515 | | draw_cmd = " ".join(cmd_args) |
| 516 | | |
| 517 | | try: |
| 518 | | subprocess.run(draw_cmd, shell=True, check=True) |
| 519 | | print(f"SUCCESS: Mapped all boxes and text labels onto -> '{output_image}'\n") |
| 520 | | except subprocess.CalledProcessError: |
| 521 | | print("ERROR: ImageMagick rendering execution failed.\n") |
| 522 | | else: |
| 523 | | print("INFO: No operational object targets were captured by the NPU context.\n") |
| 524 | | |
| 525 | | if __name__ == '__main__': |
| 526 | | main() |
| 527 | | EOF |
| 528 | | }}} |
| 529 | | - The script using PyGObject which is a Python package that provides bindings for libraries based on GObject Introspection such as GTK, !WebKit, and GStreamer. It allows you to use C-based frameworks in python. We need to install the C libs for GSTreamer for this: |
| 530 | | {{{#!bash |
| 531 | | apt-get install -y \ |
| 532 | | libcairo2-dev \ |
| 533 | | libgirepository-2.0-dev \ |
| 534 | | python3-dev \ |
| 535 | | python3-gst-1.0 \ |
| 536 | | cmake pkg-config |
| 537 | | # we are also going to need to install gstreamer and its dev packages |
| 538 | | apt-get install -y \ |
| 539 | | libgstreamer1.0-dev \ |
| 540 | | libgstreamer-plugins-base1.0-dev \ |
| 541 | | libgstreamer-plugins-bad1.0-dev \ |
| 542 | | gstreamer1.0-plugins-base \ |
| 543 | | gstreamer1.0-plugins-good \ |
| 544 | | gstreamer1.0-plugins-bad \ |
| 545 | | gstreamer1.0-plugins-ugly \ |
| 546 | | gstreamer1.0-libav \ |
| 547 | | gstreamer1.0-tools |
| 548 | | }}} |
| 549 | | - create a python virtual env (always a good idea to keep python dependencies containerized) and install python libs we need: |
| 550 | | {{{#!bash |
| 551 | | # create a venv (.venv) |
| 552 | | uv venv |
| 553 | | # install our scripts dependencies |
| 554 | | uv pip install pygobject |
| 555 | | }}} |
| 556 | | - (optional) fetch some images for detection |
| 557 | | {{{#!bash |
| 558 | | # fetch a coco validation image; it contains a dog on a bench and the dog is at 208,147 to 293,289 |
| 559 | | wget http://images.cocodataset.org/val2017/000000546829.jpg -O dog.jpg |
| 560 | | # use ffmpeg to grab a frame from within an MP4 |
| 561 | | apt install -y ffmpeg |
| 562 | | ffmpeg -i /usr/share/ara2-vision-examples/sample_videos/video_0.mp4 -f null - # shows how lon git is (time=00:00:15.50) |
| 563 | | ffmpeg -i /usr/share/ara2-vision-examples/sample_videos/video_0.mp4 -ss 00:00:5 -frames:v 1 traffic.png |
| 564 | | }}} |
| 565 | | - run the script (image_detect.py <source-image> <destination-image> [model-path]) |
| 566 | | {{{#!bash |
| 567 | | uv run image_detect.py dog.jpg coco_detections.jpg |
| 568 | | }}} |
| 569 | | - Note that without shm the pipeline needs to copy the raw image bytes over a local network-style socket connection. By mounting a dedicated memory path to /dev/shm you can eliminate that transfer (zero-copy): dvPre dumps the processed directly into a designated block of system RAM and dvInf uses a pointer to it |
| 570 | | - you would think that if your original image was 1080x1920 and you resized it to the model size of 640x640 that if you tell dvPost the orig-width=1080 orig-height=1920 that it would scale the bounding boxes properly however in practice it seems it does not unless your image has the same aspect ratio of the model. mapping it as above (telling dvPost that the image is 640x640 and scaling ourselves) resolves this |
| 571 | | - images: |
| 572 | | |
| 573 | | [[Image(dog.jpg,400px)]] |
| 574 | | [[Image(dog_detect.jpg,400px)]] |
| 575 | | |
| 576 | | [[Image(traffic.jpg,400px)]] |
| 577 | | [[Image(traffic_detect_yolo8n.jpg,400px)]] |
| 578 | | [[Image(traffic_detect_yolo8x.jpg,400px)]] |
| 579 | | |
| 580 | | |
| 581 | | * Video detection with boxing via Python in a headless webapp |
| 582 | | - Python is incredibly useful for accessing GStreamer and handling the ARA detection frame data and building webapps |
| 583 | | - The script using PyGObject which is a Python package that provides bindings for libraries based on GObject Introspection such as GTK, !WebKit, and GStreamer. It allows you to use C-based frameworks in python. We need to install the C libs for GSTreamer for this: |
| 584 | | {{{#!bash |
| 585 | | apt-get install -y \ |
| 586 | | libcairo2-dev \ |
| 587 | | libgirepository-2.0-dev \ |
| 588 | | python3-dev \ |
| 589 | | python3-gst-1.0 \ |
| 590 | | cmake pkg-config |
| 591 | | # we are also going to need to install gstreamer and its dev packages |
| 592 | | apt-get install -y \ |
| 593 | | libgstreamer1.0-dev \ |
| 594 | | libgstreamer-plugins-base1.0-dev \ |
| 595 | | libgstreamer-plugins-bad1.0-dev \ |
| 596 | | gstreamer1.0-plugins-base \ |
| 597 | | gstreamer1.0-plugins-good \ |
| 598 | | gstreamer1.0-plugins-bad \ |
| 599 | | gstreamer1.0-plugins-ugly \ |
| 600 | | gstreamer1.0-libav \ |
| 601 | | gstreamer1.0-tools |
| 602 | | }}} |
| 603 | | - create a python virtual env (always a good idea to keep python dependencies containerized) and install python libs we need: |
| 604 | | {{{#!bash |
| 605 | | # create a venv (.venv) |
| 606 | | uv venv |
| 607 | | # install our scripts dependencies |
| 608 | | uv pip install pygobject opencv-python-headless |
| 609 | | cat << EOF > vision-webapp.py |
| 610 | | #!/usr/bin/env python3 |
| 611 | | """ |
| 612 | | Ara NPU Basic Video Stream & Inference Hub |
| 613 | | ========================================== |
| 614 | | """ |
| 615 | | |
| 616 | | import argparse |
| 617 | | import ctypes |
| 618 | | import glob |
| 619 | | import os |
| 620 | | import sys |
| 621 | | import threading |
| 622 | | import time |
| 623 | | import logging |
| 624 | | import cv2 |
| 625 | | import numpy as np |
| 626 | | from flask import Flask, Response, jsonify, request, render_template_string |
| 627 | | import gi |
| 628 | | |
| 629 | | gi.require_version('Gst', '1.0') |
| 630 | | from gi.repository import Gst |
| 631 | | Gst.init(None) |
| 632 | | |
| 633 | | # Quiet down Werkzeug HTTP traffic logging to suppress 1Hz AJAX console pollution |
| 634 | | log = logging.getLogger('werkzeug') |
| 635 | | log.setLevel(logging.ERROR) |
| 636 | | |
| 637 | | app = Flask(__name__) |
| 638 | | lock = threading.Lock() |
| 639 | | |
| 640 | | class AraDetection(ctypes.Structure): |
| 641 | | _pack_ = 1 |
| 642 | | _fields_ = [ |
| 643 | | ("xmin", ctypes.c_float), ("ymin", ctypes.c_float), |
| 644 | | ("xmax", ctypes.c_float), ("ymax", ctypes.c_float), |
| 645 | | ("confidence", ctypes.c_float), ("class_id", ctypes.c_int32), |
| 646 | | ("class_name_ptr", ctypes.c_void_p) |
| 647 | | ] |
| 648 | | |
| 649 | | # --- STATE STORAGE --- |
| 650 | | STATE_REPO = { |
| 651 | | "frame": None, |
| 652 | | "detections": [], |
| 653 | | "active_source": None, |
| 654 | | "active_model_name": "yolov8n", |
| 655 | | "active_model_path": "/usr/share/cnn/detection/yolov8n/model.dvm", |
| 656 | | "restart_flag": False, |
| 657 | | "source_registry": [], |
| 658 | | "model_registry": ["yolov8n"], |
| 659 | | |
| 660 | | # Target Pipeline Resolutions |
| 661 | | "CANVAS_W": 640, |
| 662 | | "CANVAS_H": 360, |
| 663 | | "MODEL_W": 640, |
| 664 | | "MODEL_H": 640, |
| 665 | | |
| 666 | | # Live Telemetry Metrics |
| 667 | | "native_w": 0, |
| 668 | | "native_h": 0, |
| 669 | | "stream_w": 0, |
| 670 | | "stream_h": 0, |
| 671 | | "inference_fps": 0.0 |
| 672 | | } |
| 673 | | |
| 674 | | # FPS Calculation variables bound directly to the Inference thread |
| 675 | | inference_timestamps = [] |
| 676 | | |
| 677 | | COCO_LABELS = { |
| 678 | | 0: 'person', 1: 'bicycle', 2: 'car', 3: 'motorcycle', 4: 'airplane', 5: 'bus', |
| 679 | | 6: 'train', 7: 'truck', 8: 'boat', 9: 'traffic light', 10: 'fire hydrant', |
| 680 | | 11: 'stop sign', 12: 'parking meter', 13: 'bench', 14: 'bird', 15: 'cat', |
| 681 | | 16: 'dog', 17: 'horse', 18: 'sheep', 19: 'cow', 20: 'elephant', 21: 'bear', |
| 682 | | 22: 'zebra', 23: 'giraffe', 24: 'backpack', 25: 'umbrella', 26: 'handbag', |
| 683 | | 27: 'tie', 28: 'suitcase', 29: 'frisbee', 30: 'skis', 31: 'snowboard', |
| 684 | | 32: 'sports ball', 33: 'kite', 34: 'baseball bat', 35: 'baseball glove', |
| 685 | | 36: 'skateboard', 37: 'surfboard', 38: 'tennis racket', 39: 'bottle', |
| 686 | | 40: 'wine glass', 41: 'cup', 42: 'fork', 43: 'knife', 44: 'spoon', 45: 'bowl', |
| 687 | | 46: 'banana', 47: 'apple', 48: 'sandwich', 49: 'orange', 50: 'broccoli', |
| 688 | | 51: 'carrot', 52: 'hot dog', 53: 'pizza', 54: 'donut', 55: 'cake', |
| 689 | | 56: 'chair', 57: 'couch', 58: 'potted plant', 59: 'bed', 60: 'dining table', |
| 690 | | 61: 'toilet', 62: 'tv', 63: 'laptop', 64: 'mouse', 65: 'remote', 66: 'keyboard', |
| 691 | | 67: 'cell phone', 68: 'microwave', 69: 'oven', 70: 'toaster', 71: 'sink', |
| 692 | | 72: 'refrigerator', 73: 'book', 74: 'clock', 75: 'vase', 76: 'scissors', |
| 693 | | 77: 'teddy bear', 78: 'hair drier', 79: 'toothbrush' |
| 694 | | } |
| 695 | | |
| 696 | | def build_source_injection_string(source_path): |
| 697 | | if source_path.endswith(".mp4"): |
| 698 | | return f"filesrc location={source_path} ! decodebin ! videoconvert ! tee name=t " |
| 699 | | else: |
| 700 | | return f"v4l2src device={source_path} ! videoconvert ! tee name=t " |
| 701 | | |
| 702 | | def gstreamer_orchestration_loop(): |
| 703 | | global inference_timestamps |
| 704 | | CANVAS_W = STATE_REPO["CANVAS_W"] |
| 705 | | CANVAS_H = STATE_REPO["CANVAS_H"] |
| 706 | | MODEL_W = STATE_REPO["MODEL_W"] |
| 707 | | MODEL_H = STATE_REPO["MODEL_H"] |
| 708 | | |
| 709 | | while True: |
| 710 | | while STATE_REPO["active_source"] is None: |
| 711 | | time.sleep(0.2) |
| 712 | | if STATE_REPO["restart_flag"]: |
| 713 | | break |
| 714 | | |
| 715 | | current_target_source = STATE_REPO["active_source"] |
| 716 | | current_target_model = STATE_REPO["active_model_path"] |
| 717 | | STATE_REPO["restart_flag"] = False |
| 718 | | |
| 719 | | if current_target_source is None: |
| 720 | | continue |
| 721 | | |
| 722 | | source_segment = build_source_injection_string(current_target_source) |
| 723 | | |
| 724 | | pipe_str = ( |
| 725 | | f"{source_segment} " |
| 726 | | f"t. ! queue max-size-buffers=2 leaky=downstream ! appsink name=nativesink sync=false async=false emit-signals=true " |
| 727 | | f"t. ! queue max-size-buffers=2 leaky=downstream ! videoscale ! video/x-raw,width={CANVAS_W},height={CANVAS_H} ! videoconvert ! video/x-raw,format=BGR ! appsink name=framesink sync=false async=false emit-signals=true " |
| 728 | | f"t. ! queue max-size-buffers=2 leaky=downstream ! " |
| 729 | | f"videoscale ! video/x-raw,width={MODEL_W},height={MODEL_H} ! videoconvert ! video/x-raw,format=BGRA ! " |
| 730 | | f"dvPre model={current_target_model} ! " |
| 731 | | f"dvInf model={current_target_model} sock=/var/run/proxy.sock use-shm=true shm-path=/dev/shm/ara_inf_ ! " |
| 732 | | f"dvPost model={current_target_model} orig-width={MODEL_W} orig-height={MODEL_H} ! " |
| 733 | | f"appsink name=postsink sync=false async=false emit-signals=true" |
| 734 | | ) |
| 735 | | |
| 736 | | print(f"[LAUNCH PIPELINE]\n {pipe_str}\n") |
| 737 | | pipeline = Gst.parse_launch(pipe_str) |
| 738 | | |
| 739 | | native_sink = pipeline.get_by_name("nativesink") |
| 740 | | frame_sink = pipeline.get_by_name("framesink") |
| 741 | | post_sink = pipeline.get_by_name("postsink") |
| 742 | | |
| 743 | | def on_native_caps(sink): |
| 744 | | sample = sink.emit("pull-sample") |
| 745 | | if sample: |
| 746 | | caps = sample.get_caps() |
| 747 | | struct = caps.get_structure(0) |
| 748 | | STATE_REPO["native_w"] = struct.get_value("width") |
| 749 | | STATE_REPO["native_h"] = struct.get_value("height") |
| 750 | | return Gst.FlowReturn.OK |
| 751 | | |
| 752 | | def on_new_detection(sink): |
| 753 | | global inference_timestamps |
| 754 | | sample = sink.emit("pull-sample") |
| 755 | | if sample: |
| 756 | | # Calculate FPS derived purely from the inference hardware return loop |
| 757 | | now = time.time() |
| 758 | | inference_timestamps.append(now) |
| 759 | | if len(inference_timestamps) > 30: |
| 760 | | inference_timestamps.pop(0) |
| 761 | | if len(inference_timestamps) > 1: |
| 762 | | STATE_REPO["inference_fps"] = len(inference_timestamps) / (inference_timestamps[-1] - inference_timestamps[0]) |
| 763 | | |
| 764 | | buffer = sample.get_buffer() |
| 765 | | raw_bytes = buffer.extract_dup(0, buffer.get_size()) |
| 766 | | if raw_bytes and len(raw_bytes) >= 4: |
| 767 | | num_detections = np.frombuffer(raw_bytes[:4], dtype=np.uint32)[0] |
| 768 | | local_dets = [] |
| 769 | | offset = 4 |
| 770 | | ds = ctypes.sizeof(AraDetection) |
| 771 | | for _ in range(num_detections): |
| 772 | | if offset + ds > len(raw_bytes): break |
| 773 | | det = AraDetection.from_buffer_copy(raw_bytes[offset:offset+ds]) |
| 774 | | offset += ds |
| 775 | | local_dets.append((det.class_id, det.confidence, det.xmin, det.ymin, det.xmax, det.ymax)) |
| 776 | | STATE_REPO["detections"] = local_dets |
| 777 | | return Gst.FlowReturn.OK |
| 778 | | |
| 779 | | def on_new_frame(sink): |
| 780 | | sample = sink.emit("pull-sample") |
| 781 | | if sample: |
| 782 | | buffer = sample.get_buffer() |
| 783 | | caps = sample.get_caps() |
| 784 | | struct = caps.get_structure(0) |
| 785 | | w = struct.get_value("width") |
| 786 | | h = struct.get_value("height") |
| 787 | | |
| 788 | | STATE_REPO["stream_w"] = w |
| 789 | | STATE_REPO["stream_h"] = h |
| 790 | | |
| 791 | | raw_bytes = buffer.extract_dup(0, buffer.get_size()) |
| 792 | | if raw_bytes: |
| 793 | | try: |
| 794 | | frame_flat = np.frombuffer(raw_bytes, dtype=np.uint8) |
| 795 | | frame_arr = frame_flat.reshape((h, w, 3)) |
| 796 | | STATE_REPO["frame"] = frame_arr.copy() |
| 797 | | except ValueError: |
| 798 | | pass |
| 799 | | return Gst.FlowReturn.OK |
| 800 | | |
| 801 | | native_sink.connect("new-sample", on_native_caps) |
| 802 | | post_sink.connect("new-sample", on_new_detection) |
| 803 | | frame_sink.connect("new-sample", on_new_frame) |
| 804 | | pipeline.set_state(Gst.State.PLAYING) |
| 805 | | |
| 806 | | bus = pipeline.get_bus() |
| 807 | | while True: |
| 808 | | msg = bus.timed_pop_filtered(Gst.SECOND * 0.05, Gst.MessageType.ERROR | Gst.MessageType.EOS) |
| 809 | | if msg: |
| 810 | | if msg.type == Gst.MessageType.EOS and current_target_source.endswith(".mp4"): |
| 811 | | pipeline.seek_simple(Gst.Format.TIME, Gst.SeekFlags.FLUSH | Gst.SeekFlags.KEY_UNIT, 0) |
| 812 | | continue |
| 813 | | break |
| 814 | | |
| 815 | | if STATE_REPO["restart_flag"]: |
| 816 | | break |
| 817 | | |
| 818 | | pipeline.set_state(Gst.State.NULL) |
| 819 | | STATE_REPO["frame"] = None |
| 820 | | STATE_REPO["detections"] = [] |
| 821 | | STATE_REPO["native_w"] = 0 |
| 822 | | STATE_REPO["native_h"] = 0 |
| 823 | | STATE_REPO["stream_w"] = 0 |
| 824 | | STATE_REPO["stream_h"] = 0 |
| 825 | | STATE_REPO["inference_fps"] = 0.0 |
| 826 | | inference_timestamps = [] |
| 827 | | time.sleep(1.0) |
| 828 | | |
| 829 | | @app.route('/') |
| 830 | | def index(): |
| 831 | | src_active = STATE_REPO["active_source"] |
| 832 | | |
| 833 | | if not STATE_REPO["source_registry"]: |
| 834 | | src_html = '<option value="" disabled selected>-- NO VALID INPUT SOURCES AVAILABLE --</option>' |
| 835 | | else: |
| 836 | | src_html = '<option value="" disabled selected>-- SELECT TARGET SOURCE CHANNEL --</option>' if src_active is None else "" |
| 837 | | src_html += "".join(f'<option value="{s}" {"selected" if s == src_active else ""}>{s}</option>' for s in STATE_REPO["source_registry"]) |
| 838 | | |
| 839 | | mdl_active = STATE_REPO["active_model_name"] |
| 840 | | mdl_html = "".join(f'<option value="{m}" {"selected" if m == mdl_active else ""}>{m}</option>' for m in STATE_REPO["model_registry"]) |
| 841 | | |
| 842 | | html_template = """<!DOCTYPE html> |
| 843 | | <html> |
| 844 | | <head> |
| 845 | | <title>Ara Stream Client</title> |
| 846 | | <style> |
| 847 | | body { font-family: sans-serif; background: #0c0c0e; color: #e1e1e6; margin: 0; padding: 20px; display: flex; flex-direction: column; align-items: center; } |
| 848 | | .dashboard-layout { display: flex; flex-direction: column; gap: 15px; width: 660px; } |
| 849 | | .panel { background: #121216; padding: 12px 15px; border-radius: 6px; border: 1px solid #1f1f24; display: flex; flex-direction: column; gap: 10px; } |
| 850 | | .control-row { display: flex; align-items: center; justify-content: space-between; } |
| 851 | | label { font-size: 12px; font-weight: bold; color: #8f8f9d; text-transform: uppercase; } |
| 852 | | select { background: #0c0c0e; color: #fff; border: 1px solid #04d361; padding: 6px 10px; border-radius: 4px; width: 420px; outline: none; } |
| 853 | | .stats-banner { display: flex; justify-content: space-between; background: #17171f; padding: 10px 15px; border: 1px solid #1f1f24; border-radius: 4px; font-family: monospace; font-size: 13px; color: #8f8f9d; } |
| 854 | | .stats-banner span strong { color: #04d361; } |
| 855 | | .media-container { background: #121216; padding: 8px; border-radius: 6px; border: 1px solid #1f1f24; position: relative; min-height: 480px; display: flex; align-items: center; justify-content: center; } |
| 856 | | img { display: block; border-radius: 4px; width: 100%; height: auto; } |
| 857 | | .overlay { position: absolute; top: 0; left: 0; width: 100%; height: 100%; background: rgba(12,12,14,0.9); display: flex; flex-direction: column; align-items: center; justify-content: center; border-radius: 6px; text-align: center; } |
| 858 | | .prompt-text { color: #04d361; font-weight: bold; font-size: 16px; margin-bottom: 10px; } |
| 859 | | </style> |
| 860 | | <script> |
| 861 | | let streamStarted = {% if active_src %}true{% else %}false{% endif %}; |
| 862 | | |
| 863 | | async function switchConfig() { |
| 864 | | const src = document.getElementById('source-picker').value; |
| 865 | | const mdl = document.getElementById('model-picker').value; |
| 866 | | if(!src) return; |
| 867 | | |
| 868 | | await fetch('/api/swap_config', { |
| 869 | | method: 'POST', |
| 870 | | headers: { 'Content-Type': 'application/json' }, |
| 871 | | body: JSON.stringify({ "source": src, "model": mdl }) |
| 872 | | }); |
| 873 | | |
| 874 | | streamStarted = true; |
| 875 | | document.getElementById('gatekeeper-overlay').style.display = 'none'; |
| 876 | | setTimeout(() => { |
| 877 | | document.getElementById('stream-player').src = '/stream.mjpg'; |
| 878 | | }, 1000); |
| 879 | | } |
| 880 | | |
| 881 | | async function updateStreamMetrics() { |
| 882 | | if (!streamStarted) return; |
| 883 | | try { |
| 884 | | const response = await fetch('/api/stream_info'); |
| 885 | | const data = await response.json(); |
| 886 | | |
| 887 | | document.getElementById('metric-res').innerText = 'Source:' + data.native_w + 'x' + data.native_h + ' Canvas:' + data.width + 'x' + data.height; |
| 888 | | document.getElementById('metric-fps').innerText = data.fps.toFixed(1); |
| 889 | | document.getElementById('metric-dets').innerText = data.detections; |
| 890 | | } catch (err) {} |
| 891 | | } |
| 892 | | setInterval(updateStreamMetrics, 1000); |
| 893 | | </script> |
| 894 | | </head> |
| 895 | | <body> |
| 896 | | <h2>Ara Vision Engine</h2> |
| 897 | | <div class="dashboard-layout"> |
| 898 | | <div class="panel"> |
| 899 | | <div class="control-row"> |
| 900 | | <label for="source-picker">Media Stream Target:</label> |
| 901 | | <select id="source-picker" onchange="switchConfig()">""" + src_html + """</select> |
| 902 | | </div> |
| 903 | | <div class="control-row"> |
| 904 | | <label for="model-picker">NPU Pipeline Model:</label> |
| 905 | | <select id="model-picker" onchange="switchConfig()">""" + mdl_html + """</select> |
| 906 | | </div> |
| 907 | | </div> |
| 908 | | |
| 909 | | <div class="stats-banner"> |
| 910 | | <span id="metric-res">Source:0x0 Canvas:0x0</span> |
| 911 | | <span>NPU Inference: <span id="metric-fps">0.0</span> FPS</span> |
| 912 | | <span>Active Detections: <span id="metric-dets">0</span></span> |
| 913 | | </div> |
| 914 | | |
| 915 | | <div class="media-container"> |
| 916 | | {% if not active_src %} |
| 917 | | <div class="overlay" id="gatekeeper-overlay"> |
| 918 | | <div class="prompt-text">Awaiting Source Context</div> |
| 919 | | <div style="color: #8f8f9d; font-size: 13px; max-width: 400px;">Please select a media path and model from the drop-downs above to mount your pipeline.</div> |
| 920 | | </div> |
| 921 | | {% endif %} |
| 922 | | <img id="stream-player" {% if active_src %}src="/stream.mjpg"{% endif %} style="max-width: """ + str(STATE_REPO["CANVAS_W"]) + """px;" /> |
| 923 | | </div> |
| 924 | | </div> |
| 925 | | </body> |
| 926 | | </html>""" |
| 927 | | return render_template_string(html_template, active_src=src_active) |
| 928 | | |
| 929 | | @app.route('/api/stream_info') |
| 930 | | def stream_info(): |
| 931 | | with lock: |
| 932 | | return jsonify({ |
| 933 | | "native_w": STATE_REPO["native_w"], |
| 934 | | "native_h": STATE_REPO["native_h"], |
| 935 | | "width": STATE_REPO["stream_w"], |
| 936 | | "height": STATE_REPO["stream_h"], |
| 937 | | "fps": STATE_REPO["inference_fps"], |
| 938 | | "detections": len(STATE_REPO["detections"]) |
| 939 | | }) |
| 940 | | |
| 941 | | @app.route('/api/swap_config', methods=['POST']) |
| 942 | | def swap_config(): |
| 943 | | payload = request.get_json() |
| 944 | | src_selected = payload.get("source") |
| 945 | | mdl_selected = payload.get("model") |
| 946 | | |
| 947 | | with lock: |
| 948 | | trigger_restart = False |
| 949 | | if src_selected in STATE_REPO["source_registry"] and STATE_REPO["active_source"] != src_selected: |
| 950 | | STATE_REPO["active_source"] = src_selected |
| 951 | | trigger_restart = True |
| 952 | | if mdl_selected in STATE_REPO["model_registry"] and STATE_REPO["active_model_name"] != mdl_selected: |
| 953 | | base_dir = app.config["MODEL_DIR"] |
| 954 | | STATE_REPO["active_model_name"] = mdl_selected |
| 955 | | STATE_REPO["active_model_path"] = os.path.join(base_dir, mdl_selected, "model.dvm") |
| 956 | | trigger_restart = True |
| 957 | | if trigger_restart: |
| 958 | | STATE_REPO["restart_flag"] = True |
| 959 | | return jsonify({"status": "success"}) |
| 960 | | |
| 961 | | def generate_mjpeg_stream_generator(): |
| 962 | | MODEL_W = float(STATE_REPO["MODEL_W"]) |
| 963 | | MODEL_H = float(STATE_REPO["MODEL_H"]) |
| 964 | | |
| 965 | | while True: |
| 966 | | time.sleep(0.04) |
| 967 | | frame_copy = STATE_REPO["frame"] |
| 968 | | local_dets = list(STATE_REPO["detections"]) |
| 969 | | if frame_copy is not None: |
| 970 | | frame = frame_copy.copy() |
| 971 | | h_native, w_native, _ = frame_copy.shape |
| 972 | | for class_id, confidence, rx1, ry1, rx2, ry2 in local_dets: |
| 973 | | cx1 = int(rx1 * (float(w_native) / MODEL_W)) |
| 974 | | cx2 = int(rx2 * (float(w_native) / MODEL_W)) |
| 975 | | cy1 = int(ry1 * (float(h_native) / MODEL_H)) |
| 976 | | cy2 = int(ry2 * (float(h_native) / MODEL_H)) |
| 977 | | label = f"{COCO_LABELS.get(class_id, f'Class {class_id}')} ({confidence*100:.1f}%)" |
| 978 | | cv2.rectangle(frame, (cx1, cy1), (cx2, cy2), (0, 255, 97), 2) |
| 979 | | cv2.putText(frame, label, (cx1, max(15, cy1 - 5)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 97), 2) |
| 980 | | _, encoded_img = cv2.imencode(".jpg", frame) |
| 981 | | yield (b'--frame\r\n' |
| 982 | | b'Content-Type: image/jpeg\r\n\r\n' + encoded_img.tobytes() + b'\r\n') |
| 983 | | else: |
| 984 | | waiting_canvas = np.zeros((480, 640, 3), dtype=np.uint8) |
| 985 | | cv2.putText(waiting_canvas, "AWAITING MEDIA INPUT SELECTION...", (140, 240), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 97), 1) |
| 986 | | _, encoded_img = cv2.imencode(".jpg", waiting_canvas) |
| 987 | | yield (b'--frame\r\n' |
| 988 | | b'Content-Type: image/jpeg\r\n\r\n' + encoded_img.tobytes() + b'\r\n') |
| 989 | | |
| 990 | | @app.route('/stream.mjpg') |
| 991 | | def video_feed_stream_route(): |
| 992 | | return Response(generate_mjpeg_stream_generator(), mimetype='multipart/x-mixed-replace; boundary=frame') |
| 993 | | |
| 994 | | def main(): |
| 995 | | parser = argparse.ArgumentParser(description="Wiki Template: Ara Flask Video Engine") |
| 996 | | parser.add_argument("--camera", default=None, help="Camera context device node path") |
| 997 | | parser.add_argument("--mp4", default=None, help="Directory containing target mp4 sample videos") |
| 998 | | parser.add_argument("--port", type=int, default=8080, help="Target port mapping") |
| 999 | | parser.add_argument("--model-dir", default="/usr/share/cnn/detection", help="Directory containing target models") |
| 1000 | | parser.add_argument("--model", default="yolov8n", help="Initial model selection") |
| 1001 | | args = parser.parse_args() |
| 1002 | | |
| 1003 | | app.config["MODEL_DIR"] = args.model_dir |
| 1004 | | STATE_REPO["source_registry"] = [] |
| 1005 | | |
| 1006 | | if args.camera and os.path.exists(args.camera): |
| 1007 | | STATE_REPO["source_registry"].append(args.camera) |
| 1008 | | |
| 1009 | | if args.mp4 and os.path.exists(args.mp4): |
| 1010 | | local_videos = glob.glob(os.path.join(args.mp4, "*.mp4")) |
| 1011 | | for vid in sorted(local_videos): |
| 1012 | | STATE_REPO["source_registry"].append(vid) |
| 1013 | | |
| 1014 | | if os.path.exists(args.model_dir): |
| 1015 | | discovered_models = [] |
| 1016 | | for entry in sorted(os.listdir(args.model_dir)): |
| 1017 | | full_subdir = os.path.join(args.model_dir, entry) |
| 1018 | | if os.path.isdir(full_subdir) and os.path.exists(os.path.join(full_subdir, "model.dvm")): |
| 1019 | | discovered_models.append(entry) |
| 1020 | | if discovered_models: |
| 1021 | | STATE_REPO["model_registry"] = discovered_models |
| 1022 | | STATE_REPO["active_model_name"] = args.model if args.model in discovered_models else discovered_models[0] |
| 1023 | | STATE_REPO["active_model_path"] = os.path.join(args.model_dir, STATE_REPO["active_model_name"], "model.dvm") |
| 1024 | | |
| 1025 | | threading.Thread(target=gstreamer_orchestration_loop, daemon=True).start() |
| 1026 | | |
| 1027 | | print(f"Server serving on: http://localhost:{args.port}/") |
| 1028 | | app.run(host='0.0.0.0', port=args.port, threaded=True, use_reloader=False, debug=False) |
| 1029 | | |
| 1030 | | if __name__ == '__main__': |
| 1031 | | main() |
| 1032 | | EOF |
| 1033 | | }}} |
| 1034 | | - run the script (vison-webapp.py [--port <portno>] [--camera <camera-dev>] [--mp4 <mp4-dir>] |
| 1035 | | {{{#!bash |
| 1036 | | uv run vision-webapp.py --camera /dev/video_webcam --mp4 /usr/share/ara2-vision-examples/sample_videos/ |
| 1037 | | }}} |
| 1038 | | - you can provide a webcam device path to enable streaming from a webcam and/or an mp4 directory to enable processing those. A dropdown will allow you to select the input stream and the model and the browser window will show you detections and statistics |
| 1039 | | |
| 1040 | | [[Image(vision-webapp.jpg,400px)]] |
| | 276 | filesrc location=traffic.png ! \ |
| | 277 | pngdec ! imagefreeze num-buffers=10 ! \ |
| | 278 | videoscale ! videoconvert ! video/x-raw,format=BGRx,width=640,height=480 ! \ |
| | 279 | dvInf model=$MODEL orig-width=640 orig-height=480 stream=0 \ |
| | 280 | sock=/var/run/proxy.sock use-shm=true shm-path=/dev/shm/ara_shm ! \ |
| | 281 | fakesink sync=false | grep Detected |
| | 282 | }}} |
| | 283 | |
| | 284 | For a more complete example see below |
| 1395 | | webchat.py: |
| 1396 | | {{{#!python |
| 1397 | | import sys |
| 1398 | | import os |
| 1399 | | from datetime import datetime |
| 1400 | | |
| 1401 | | # --- KINARA SDK PATH INJECTION --- |
| 1402 | | DVAPI_DIR = "/usr/share/rt-sdk-ara240_2.0.4/include" |
| 1403 | | if os.path.exists(DVAPI_DIR): |
| 1404 | | sys.path.append(DVAPI_DIR) |
| 1405 | | |
| 1406 | | import streamlit as st |
| 1407 | | import requests |
| 1408 | | import json |
| 1409 | | import time |
| 1410 | | import psutil |
| 1411 | | import threading |
| 1412 | | import argparse |
| 1413 | | |
| 1414 | | # Attempt to import the Kinara Python APIs |
| 1415 | | try: |
| 1416 | | from dvapi import DVSession, dv_endpoint_get_statistics, dv_endpoint_free_statistics |
| 1417 | | except ImportError: |
| 1418 | | st.error(f"Critical: dvapi.py not found at {DVAPI_DIR}") |
| 1419 | | st.stop() |
| 1420 | | |
| 1421 | | # --- ARGUMENT PARSING --- |
| 1422 | | parser = argparse.ArgumentParser() |
| 1423 | | parser.add_argument("--host", type=str, default="127.0.0.1", help="AAF Connector Host") |
| 1424 | | parser.add_argument("--port", type=str, default="8000", help="AAF Connector Port") |
| 1425 | | parser.add_argument("--proxy-sock", type=str, default="/var/run/proxy.sock", help="Kinara Proxy socket") |
| 1426 | | args, _ = parser.parse_known_args() |
| 1427 | | |
| 1428 | | # --- CONFIGURATION --- |
| 1429 | | MODEL_NAME = "Qwen2.5-7B-Instruct" |
| 1430 | | API_URL = f"http://{args.host}:{args.port}/v1/chat/completions" |
| 1431 | | LOGO_URL = "/root/gateworks_logo.png" |
| 1432 | | |
| 1433 | | # --- HARDWARE TELEMETRY HELPERS --- |
| 1434 | | def get_dvapi_npu_stats(): |
| 1435 | | try: |
| 1436 | | ret, session = DVSession.create_via_unix_socket(args.proxy_sock) |
| 1437 | | if ret != 0: return None |
| 1438 | | with session: |
| 1439 | | ret, ep_list = session.get_endpoint_list() |
| 1440 | | if ret != 0 or not ep_list: return None |
| 1441 | | ret, stats_ptr, count = dv_endpoint_get_statistics(session._session, ep_list[0]._endpoint) |
| 1442 | | if ret == 0 and count.value > 0: |
| 1443 | | s = stats_ptr[0] |
| 1444 | | TOTAL_CAPACITY_GB = 16.0 |
| 1445 | | free_gb = s.ep_dram_stats.ep_total_free_size / 1073741824 |
| 1446 | | used_gb = max(0, TOTAL_CAPACITY_GB - free_gb) |
| 1447 | | dram_pct = (used_gb / TOTAL_CAPACITY_GB) * 100 |
| 1448 | | is_busy = st._npu_lock.locked() |
| 1449 | | data = {"temp": s.ep_temp, "util": 100 if is_busy else 0, "ram_pct": dram_pct} |
| 1450 | | dv_endpoint_free_statistics(stats_ptr, count) |
| 1451 | | return data |
| 1452 | | except: return None |
| 1453 | | |
| 1454 | | def get_system_thermals(): |
| 1455 | | zones = [] |
| 1456 | | try: |
| 1457 | | for zone in sorted(os.listdir("/sys/class/thermal/")): |
| 1458 | | if zone.startswith("thermal_zone"): |
| 1459 | | with open(f"/sys/class/thermal/{zone}/temp", "r") as f: |
| 1460 | | z_temp = int(f.read().strip()) / 1000.0 |
| 1461 | | zones.append(z_temp) |
| 1462 | | except: pass |
| 1463 | | return zones |
| 1464 | | |
| 1465 | | def build_sidebar_html(): |
| 1466 | | n_stats = get_dvapi_npu_stats() |
| 1467 | | cpu_usage = psutil.cpu_percent() |
| 1468 | | sys_ram = psutil.virtual_memory().percent |
| 1469 | | thermals = get_system_thermals() |
| 1470 | | |
| 1471 | | npu_html = f"<div style='border-top:1px solid #444; padding-top:5px; font-size:0.82rem;'><b>🔥 Ara2 NPU</b><br>" |
| 1472 | | if n_stats: |
| 1473 | | npu_html += f"NPU: {n_stats['util']}% {n_stats['temp']:.1f}C | RAM: {n_stats['ram_pct']:.1f}%" |
| 1474 | | else: |
| 1475 | | npu_html += "NPU Telemetry Unavailable" |
| 1476 | | npu_html += "</div>" |
| 1477 | | |
| 1478 | | sys_html = f"<div style='border-top:1px solid #444; margin-top:8px; padding-top:5px; font-size:0.82rem;'><b>💻 Syst |
| 1479 | | m</b><br>" |
| 1480 | | temp_str = "/".join([f"{t:.1f}C" for t in thermals]) |
| 1481 | | sys_html += f"CPU: {cpu_usage:.1f}% {temp_str} | RAM: {sys_ram:.1f}%</div>" |
| 1482 | | |
| 1483 | | perf_val = st.session_state.get('last_perf', 'N/A') |
| 1484 | | perf_html = f"<div style='border-top:1px solid #444; margin-top:8px; padding-top:5px; font-size:0.82rem;'><b>⚡ Las |
| 1485 | | Result</b><br>{perf_val}</div>" |
| 1486 | | return npu_html + sys_html + perf_html |
| 1487 | | |
| 1488 | | # --- GLOBAL STATE --- |
| 1489 | | if not hasattr(st, '_npu_lock'): st._npu_lock = threading.Lock() |
| 1490 | | if not hasattr(st, '_active_user'): st._active_user = "None" |
| 1491 | | |
| 1492 | | st.set_page_config(page_title="Gateworks Venice AI", layout="wide") |
| 1493 | | |
| 1494 | | # --- SIDEBAR --- |
| 1495 | | with st.sidebar: |
| 1496 | | try: st.image(LOGO_URL, width=220) |
| 1497 | | except: st.write("### Gateworks Venice") |
| 1498 | | |
| 1499 | | status_slot = st.empty() |
| 1500 | | # Simplified to just show the IP address |
| 1501 | | user_id = st.context.ip_address or "127.0.0.1" |
| 1502 | | |
| 1503 | | if st._npu_lock.locked(): |
| 1504 | | status_slot.warning(f"⚠️ BUSY: {st._active_user}") |
| 1505 | | else: |
| 1506 | | status_slot.success("🟢 READY") |
| 1507 | | |
| 1508 | | st.caption(f"User: {user_id}") |
| 1509 | | |
| 1510 | | stats_slot = st.empty() |
| 1511 | | stats_slot.markdown(build_sidebar_html(), unsafe_allow_html=True) |
| 1512 | | |
| 1513 | | # --- MAIN INTERFACE --- |
| 1514 | | st.title("🤖 i.MX Edge LLM") |
| 1515 | | |
| 1516 | | if "messages" not in st.session_state: st.session_state.messages = [] |
| 1517 | | for msg in st.session_state.messages: |
| 1518 | | with st.chat_message(msg["role"]): st.markdown(msg["content"]) |
| 1519 | | |
| 1520 | | if prompt := st.chat_input("Ask the NPU..."): |
| 1521 | | st.chat_message("user").markdown(prompt) |
| 1522 | | st.session_state.messages.append({"role": "user", "content": prompt}) |
| 1523 | | |
| 1524 | | # Console: Log the Incoming Request / Queue status |
| 1525 | | ts_in = datetime.now().strftime("%H:%M:%S") |
| 1526 | | print(f"[{ts_in}] QUEUED: Request from {user_id} -> '{prompt[:40]}...'") |
| 1527 | | |
| 1528 | | with st.chat_message("assistant"): |
| 1529 | | response_placeholder = st.empty() |
| 1530 | | |
| 1531 | | # This lock handles the "Queued" logic—it will block here if someone else is talking |
| 1532 | | with st._npu_lock: |
| 1533 | | st._active_user = user_id |
| 1534 | | status_slot.warning(f"⚠️ BUSY: {user_id}") |
| 1535 | | |
| 1536 | | ts_start = datetime.now().strftime("%H:%M:%S") |
| 1537 | | print(f"[{ts_start}] PROCESSING: Active inference for {user_id}") |
| 1538 | | |
| 1539 | | full_response, token_count, start_time = "", 0, time.time() |
| 1540 | | |
| 1541 | | try: |
| 1542 | | payload = {"model": MODEL_NAME, "messages": st.session_state.messages, "stream": True} |
| 1543 | | r = requests.post(API_URL, json=payload, stream=True, timeout=120) |
| 1544 | | |
| 1545 | | for line in r.iter_lines(): |
| 1546 | | if line: |
| 1547 | | decoded = line.decode('utf-8').replace('data: ', '') |
| 1548 | | if decoded.strip() == "[DONE]": break |
| 1549 | | try: |
| 1550 | | chunk = json.loads(decoded) |
| 1551 | | content = chunk["choices"][0]["delta"].get("content", "") |
| 1552 | | if content: |
| 1553 | | full_response += content |
| 1554 | | token_count += 1 |
| 1555 | | response_placeholder.markdown(full_response + "▌") |
| 1556 | | |
| 1557 | | if token_count % 12 == 0: |
| 1558 | | stats_slot.markdown(build_sidebar_html(), unsafe_allow_html=True) |
| 1559 | | except: continue |
| 1560 | | |
| 1561 | | duration = time.time() - start_time |
| 1562 | | tps = token_count / duration if duration > 0 else 0 |
| 1563 | | st.session_state.last_perf = f"{token_count} tokens @ {tps:.1f} t/s" |
| 1564 | | |
| 1565 | | response_placeholder.markdown(full_response) |
| 1566 | | st.session_state.messages.append({"role": "assistant", "content": full_response}) |
| 1567 | | |
| 1568 | | # Console: Log Completion |
| 1569 | | ts_out = datetime.now().strftime("%H:%M:%S") |
| 1570 | | print(f"[{ts_out}] COMPLETE: {user_id} | {token_count} tokens | {tps:.1f} t/s") |
| 1571 | | |
| 1572 | | except Exception as e: |
| 1573 | | st.error(f"Error: {e}") |
| 1574 | | print(f"[{datetime.now().strftime('%H:%M:%S')}] ERROR: {e}") |
| 1575 | | finally: |
| 1576 | | st._active_user = "None" |
| 1577 | | stats_slot.markdown(build_sidebar_html(), unsafe_allow_html=True) |
| 1578 | | status_slot.success("🟢 READY") |
| 1579 | | st.rerun() |
| 1580 | | }}} |
| 1581 | | |
| 1582 | | Execution: |
| 1583 | | {{{#!bash |
| 1584 | | $ mkdir /root/webapp |
| 1585 | | $ cd /root/webapp |
| 1586 | | $ uv venv # create virtual python env in current dir |
| 1587 | | $ uv pip install streamlit requests psutil argparse # install python deps |
| 1588 | | $ uv run streamlit run webchat.py --server.address 0.0.0.0 --server.port 8501 -- --user-map users.json --host 127.0.0.1 --port 8000 |
| 1589 | | }}} |
| 1590 | | |
| 1591 | | Service: |
| 1592 | | - if want this to run as a service: |
| 1593 | | {{{#!bash |
| 1594 | | cat << EOF > /etc/systemd/system/eiq-webapp.service: |
| 1595 | | [Unit] |
| 1596 | | Description=Streamlit Webapp for eIQ AAF |
| 1597 | | # Start after network is up |
| 1598 | | After=network.target |
| 1599 | | # We don't use 'After=eiq-aaf-connector.service' to avoid potential boot cycles |
| 1600 | | StartLimitIntervalSec=0 |
| 1601 | | |
| 1602 | | [Service] |
| 1603 | | Type=simple |
| 1604 | | User=root |
| 1605 | | # Ensure we are in the directory where webapp.py lives |
| 1606 | | WorkingDirectory=/root/webapp |
| 1607 | | |
| 1608 | | # 1. Wait until the Connector is actually listening on Port 8000 |
| 1609 | | ExecStartPre=/bin/bash -c 'until ss -Hltn | grep -E -q ":8000([[:space:]]|$)"; do echo "Waiting for eIQ Connector on Port 8000..." >&2; sleep 5; done' |
| 1610 | | |
| 1611 | | # 2. Launch the app using uv |
| 1612 | | # Note: Using absolute path for uv is safer in systemd |
| 1613 | | ExecStart=/usr/local/bin/uv run streamlit run webapp.py \ |
| 1614 | | --server.address 0.0.0.0 \ |
| 1615 | | --server.port 8501 \ |
| 1616 | | -- \ |
| 1617 | | --user-map users.json \ |
| 1618 | | --host 127.0.0.1 \ |
| 1619 | | --port 8000 |
| 1620 | | |
| 1621 | | # Restart logic |
| 1622 | | Restart=on-failure |
| 1623 | | RestartSec=10s |
| 1624 | | StartLimitBurst=0 |
| 1625 | | |
| 1626 | | # Standard Logging |
| 1627 | | StandardOutput=journal |
| 1628 | | StandardError=journal |
| 1629 | | |
| 1630 | | [Install] |
| 1631 | | WantedBy=multi-user.target |
| 1632 | | EOF |
| 1633 | | systemctl daemon-reload |
| 1634 | | systemctl enable eiq-webapp.service |
| 1635 | | systemctl start eiq-webapp.service |
| 1636 | | }}} |
| | 632 | 1. create a python virtual env (always a good idea to keep python dependencies containerized) and install python libs we need: |
| | 633 | {{{#!bash |
| | 634 | # create a dir for the venv |
| | 635 | mkdir webchat |
| | 636 | cd webchat |
| | 637 | # create a venv (.venv) |
| | 638 | uv venv |
| | 639 | # install our scripts dependencies |
| | 640 | uv pip install -q fastapi psutil uvicorn |
| | 641 | }}} |
| | 642 | 1. fetch the script |
| | 643 | {{{#!bash |
| | 644 | wget https://dev.gateworks.com/ara/examples/webchat.py |
| | 645 | }}} |
| | 646 | 1. run the script |
| | 647 | {{{#!bash |
| | 648 | uv run webchat.py |
| | 649 | }}} |
| | 650 | 1. Open a web browser to your boards IP address port 8080: http://<ipaddr>:8080 |
| | 651 | |
| | 652 | Notes: |
| | 653 | * By default this will listen for HTTP requests on port 8080 |
| | 654 | * On startup it will enable the model (specified in the script) and restart the eIQ server if needed - waiting for the model to load may take several minutes |
| 1647 | | Example: |
| 1648 | | - if you want some video examples you can download NXP's vlm-edge-studio_1.0.0.deb and extract its data: |
| 1649 | | {{{#!bash |
| 1650 | | # extract data (but don't install the deb) |
| 1651 | | dpkg-deb --vextract vlm-edge-studio_1.0.0.deb / |
| 1652 | | }}} |
| 1653 | | - this installs a number of videos to /usr/share/vlm-edge-studio/assets/videos |
| 1654 | | - The AAF connector requires a lot of DRAM when loading large models (ie the 12GB Qwen2.5-VL-7B-Instruct model) so we will create a swap file to avoid memory issues when loading the model: |
| 1655 | | {{{#!bash |
| 1656 | | swapon --show # shows nothing as not enabled |
| 1657 | | # pre-allocate space for swap file |
| 1658 | | fallocate -l 4G /swapfile |
| 1659 | | # make sure it is accessible by root only |
| 1660 | | chmod 600 /swapfile |
| 1661 | | # format the file as swap |
| 1662 | | mkswap /swapfile |
| 1663 | | # activate the swapfile |
| 1664 | | swapon /swapfile |
| 1665 | | # add it to /etc/fstab so that it mounts on boot |
| 1666 | | echo '/swapfile none swap sw 0 0' >> /etc/fstab |
| 1667 | | }}} |
| 1668 | | - install Qwen2.5-VL-7B-Instruct-Ara240 model |
| 1669 | | {{{#!bash |
| 1670 | | fetch_models --repo-id nxp/Qwen2.5-VL-7B-Instruct-Ara240 # 12GB |
| 1671 | | }}} |
| 1672 | | - To avoid loading models we are not using into the ARA and run into memory issues, disable all models except for Qwen2.5-7B-Instruct in the AAF connectors config file: |
| 1673 | | {{{#!python |
| 1674 | | python3 -c 'import json; p="/usr/share/eiq/aaf-connector/server_config.json"; f=open(p,"r+"); d=json.load(f); [m.update({"enabled": (m.get("name") == "Qwen2.5-VL-7B-Instruct")}) for m in d.get("available_models", [])]; f.seek(0); json.dump(d, f, indent=4); f.truncate()' |
| 1675 | | # restart AAF connector |
| 1676 | | systemctl restart eiq-aaf-connector.service |
| 1677 | | # wait for it to be up and running (as it will take several minutes to load the 12GB Qwen2.5-7B-Instruct to the ARA) |
| 1678 | | until (echo > /dev/tcp/127.0.0.1/8000) >/dev/null 2>&1; do echo -n .; sleep 1; done |
| 1679 | | }}} |
| 1680 | | - create a dir for us to work in and create the python script |
| 1681 | | {{{#!bash |
| 1682 | | mkdir vlm-webapp; cd vlm-webapp |
| 1683 | | cat << \EOF > vlm.py |
| 1684 | | import argparse |
| 1685 | | import os |
| 1686 | | import httpx |
| 1687 | | import uvicorn |
| 1688 | | import json |
| 1689 | | import urllib.request |
| 1690 | | import time |
| 1691 | | from datetime import datetime |
| 1692 | | from fastapi import FastAPI, HTTPException |
| 1693 | | from fastapi.responses import HTMLResponse, StreamingResponse |
| 1694 | | from fastapi.staticfiles import StaticFiles |
| 1695 | | from pydantic import BaseModel |
| 1696 | | from typing import List, Dict |
| 1697 | | |
| 1698 | | # ═══════════════════════════════════════════════════════════════ |
| 1699 | | # Command Line Arguments & Global Constants Configuration |
| 1700 | | # ═══════════════════════════════════════════════════════════════ |
| 1701 | | parser = argparse.ArgumentParser(description="VLM Edge Studio WebApp Bridge") |
| 1702 | | parser.add_argument("--video-dir", required=True, help="Directory path where video MP4 files are hosted") |
| 1703 | | parser.add_argument("--aaf-server", default="http://127.0.0.1:8000", help="AAF Server backend Base URL") |
| 1704 | | parser.add_argument("--host", default="0.0.0.0", help="Host binding address for this web application") |
| 1705 | | parser.add_argument("--port", type=int, default=8080, help="Port binding for this web application") |
| 1706 | | parser.add_argument("--verbose", action="store_true", default=False, help="Enable verbose raw JSON payload terminal dumping") |
| 1707 | | |
| 1708 | | args, _ = parser.parse_known_args() |
| 1709 | | |
| 1710 | | TARGET_MODEL = "Qwen2.5-VL-7B-Instruct" |
| 1711 | | |
| 1712 | | app = FastAPI(title="VLM Video Web Analyzer") |
| 1713 | | |
| 1714 | | if not os.path.isdir(args.video_dir): |
| 1715 | | raise RuntimeError(f"Provided video directory target does not exist: {args.video_dir}") |
| 1716 | | |
| 1717 | | # Mount local streaming static location directly from the primary video-dir configuration |
| 1718 | | app.mount("/stream/videos", StaticFiles(directory=args.video_dir), name="videos") |
| 1719 | | |
| 1720 | | class ChatMessage(BaseModel): |
| 1721 | | role: str |
| 1722 | | content: str |
| 1723 | | |
| 1724 | | class MultiTurnPayload(BaseModel): |
| 1725 | | video_name: str |
| 1726 | | history: List[ChatMessage] |
| 1727 | | |
| 1728 | | def get_timestamp(): |
| 1729 | | return datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] |
| 1730 | | |
| 1731 | | # ═══════════════════════════════════════════════════════════════ |
| 1732 | | # API Engine Intercept Proxy Routes |
| 1733 | | # ═══════════════════════════════════════════════════════════════ |
| 1734 | | |
| 1735 | | @app.get("/api/videos", tags=["Media"]) |
| 1736 | | async def get_available_videos(): |
| 1737 | | try: |
| 1738 | | if not os.path.exists(args.video_dir): |
| 1739 | | return [] |
| 1740 | | files = os.listdir(args.video_dir) |
| 1741 | | valid_extensions = (".mp4", ".mov", ".mkv", ".avi") |
| 1742 | | return [f for f in files if f.lower().endswith(valid_extensions)] |
| 1743 | | except Exception as e: |
| 1744 | | raise HTTPException(status_code=500, detail=str(e)) |
| 1745 | | |
| 1746 | | @app.get("/api/metrics") |
| 1747 | | async def proxy_metrics(): |
| 1748 | | headers = {"Accept": "application/json", "User-Agent": "AAFConnector/1.0"} |
| 1749 | | async with httpx.AsyncClient() as client: |
| 1750 | | try: |
| 1751 | | url = f"{args.aaf_server}/metrics/" |
| 1752 | | response = await client.get(url, params={"model_name": TARGET_MODEL}, headers=headers, timeout=3.0) |
| 1753 | | return response.json() |
| 1754 | | except Exception as e: |
| 1755 | | return { |
| 1756 | | TARGET_MODEL: { |
| 1757 | | "llm_average_token_per_second": 0.0, |
| 1758 | | "llm_first_infer_duration": 0.0, |
| 1759 | | "generated_token_num": 0 |
| 1760 | | } |
| 1761 | | } |
| 1762 | | |
| 1763 | | @app.post("/api/analyze") |
| 1764 | | async def proxy_analysis_stream(payload: MultiTurnPayload): |
| 1765 | | absolute_video_target_path = os.path.join(args.video_dir, payload.video_name) |
| 1766 | | start_time = time.time() |
| 1767 | | |
| 1768 | | if len(payload.history) > 1: |
| 1769 | | flattened_text = "Here is the conversation history context for this execution sequence:\n" |
| 1770 | | for msg in payload.history[:-1]: |
| 1771 | | label = "User Question" if msg.role == "user" else "Your Previous Response" |
| 1772 | | flattened_text += f"[{label}]: {msg.content}\n" |
| 1773 | | flattened_text += f"\n[New Follow-up Question to Answer]: {payload.history[-1].content}" |
| 1774 | | else: |
| 1775 | | flattened_text = payload.history[0].content |
| 1776 | | |
| 1777 | | aaf_payload = { |
| 1778 | | "model": TARGET_MODEL, |
| 1779 | | "stream": True, |
| 1780 | | "messages": [{ |
| 1781 | | "role": "user", |
| 1782 | | "content": [ |
| 1783 | | {"type": "text", "text": flattened_text}, |
| 1784 | | {"type": "video_url", "video_url": {"url": absolute_video_target_path}} |
| 1785 | | ] |
| 1786 | | }] |
| 1787 | | } |
| 1788 | | |
| 1789 | | print("\n" + "═"*70) |
| 1790 | | print(f"[{get_timestamp()}] [CONVERSATIONAL INFERENCE DISPATCH]") |
| 1791 | | print(f" Model ID : {TARGET_MODEL}") |
| 1792 | | print(f" Target Path : {absolute_video_target_path}") |
| 1793 | | print(f" Turn Count : {len(payload.history)} turns processed in session state.") |
| 1794 | | |
| 1795 | | if args.verbose: |
| 1796 | | print("─"*70) |
| 1797 | | print("[RAW OUTGOING JSON PAYLOAD SENT TO AAF SERVER]:") |
| 1798 | | print(json.dumps(aaf_payload, indent=2)) |
| 1799 | | |
| 1800 | | print("═"*70 + "\n") |
| 1801 | | |
| 1802 | | def raw_socket_generator(): |
| 1803 | | target_endpoint = f"{args.aaf_server}/v1/chat/completions" |
| 1804 | | data_bytes = json.dumps(aaf_payload).encode('utf-8') |
| 1805 | | |
| 1806 | | req = urllib.request.Request( |
| 1807 | | target_endpoint, |
| 1808 | | data=data_bytes, |
| 1809 | | headers={ |
| 1810 | | "Content-Type": "application/json", |
| 1811 | | "Accept": "application/json", |
| 1812 | | "User-Agent": "AAFConnector/1.0" |
| 1813 | | }, |
| 1814 | | method="POST" |
| 1815 | | ) |
| 1816 | | |
| 1817 | | first_token_received = False |
| 1818 | | |
| 1819 | | try: |
| 1820 | | with urllib.request.urlopen(req, timeout=300.0) as response: |
| 1821 | | while True: |
| 1822 | | line_bytes = response.readline() |
| 1823 | | if not line_bytes: |
| 1824 | | break |
| 1825 | | |
| 1826 | | line_str = line_bytes.decode('utf-8', errors='ignore') |
| 1827 | | trimmed = line_str.strip() |
| 1828 | | |
| 1829 | | if trimmed: |
| 1830 | | yield f"{trimmed}\n".encode('utf-8') |
| 1831 | | |
| 1832 | | if trimmed.startswith('data: '): |
| 1833 | | data_content = trimmed[5:].strip() |
| 1834 | | if data_content == "[DONE]": |
| 1835 | | continue |
| 1836 | | |
| 1837 | | try: |
| 1838 | | parsed = json.loads(data_content) |
| 1839 | | token = parsed["choices"][0]["delta"].get("content", "") |
| 1840 | | if token: |
| 1841 | | if not first_token_received: |
| 1842 | | ttft_duration = time.time() - start_time |
| 1843 | | print(f"[{get_timestamp()}] [TTFT / DECODE PHASE]: {ttft_duration:.2f}s.") |
| 1844 | | print(f"[{get_timestamp()}] [STREAMING TEXT TOKENS]: ", end="") |
| 1845 | | first_token_received = True |
| 1846 | | |
| 1847 | | print(token, end="", flush=True) |
| 1848 | | except Exception: |
| 1849 | | pass |
| 1850 | | |
| 1851 | | except urllib.error.HTTPError as http_err: |
| 1852 | | err_body = http_err.read().decode('utf-8', errors='ignore') |
| 1853 | | yield f"data: {{\"error\": \"AAF Server Engine error {http_err.code}: {err_body}\"}}\n\n".encode('utf-8') |
| 1854 | | except Exception as e: |
| 1855 | | yield f"data: {{\"error\": \"Direct socket pipeline fault: {str(e)}\"}}\n\n".encode('utf-8') |
| 1856 | | finally: |
| 1857 | | duration = time.time() - start_time |
| 1858 | | print("\n" + "═"*70) |
| 1859 | | print(f"[{get_timestamp()}] [INFERENCE COMPLETED] Turn Runtime: {duration:.2f}s") |
| 1860 | | print("═"*70 + "\n") |
| 1861 | | |
| 1862 | | return StreamingResponse(raw_socket_generator(), media_type="text/event-stream") |
| 1863 | | |
| 1864 | | # ═══════════════════════════════════════════════════════════════ |
| 1865 | | # User Interface (HTML Layer) |
| 1866 | | # ═══════════════════════════════════════════════════════════════ |
| 1867 | | @app.get("/", response_class=HTMLResponse) |
| 1868 | | async def serve_index(): |
| 1869 | | # Enforcing a raw python string (r"") so Python never converts or drops backslashes |
| 1870 | | html_content = r""" |
| 1871 | | <!DOCTYPE html> |
| 1872 | | <html lang="en"> |
| 1873 | | <head> |
| 1874 | | <meta charset="UTF-8"> |
| 1875 | | <title>VLM Edge Studio Analyzer</title> |
| 1876 | | <script src="https://cdn.tailwindcss.com"></script> |
| 1877 | | <style> |
| 1878 | | .skeleton-pulse { |
| 1879 | | background: linear-gradient(-90deg, #1e293b 0%, #334155 50%, #1e293b 100%); |
| 1880 | | background-size: 400% 400%; |
| 1881 | | animation: pulse 1.5s ease-in-out infinite; |
| 1882 | | } |
| 1883 | | @keyframes pulse { |
| 1884 | | 0% { background-position: 100% 50%; } |
| 1885 | | 100% { background-position: 0% 50%; } |
| 1886 | | } |
| 1887 | | </style> |
| 1888 | | </head> |
| 1889 | | <body class="bg-gray-900 text-gray-100 min-h-screen p-6"> |
| 1890 | | <div class="max-w-6xl mx-auto space-y-6"> |
| 1891 | | <header class="border-b border-gray-800 pb-4 flex justify-between items-center"> |
| 1892 | | <div> |
| 1893 | | <h1 class="text-2xl font-bold tracking-wide text-indigo-400">VLM Edge Platform Interface</h1> |
| 1894 | | <p id="metricsPanel" class="text-xs text-gray-400 mt-1 font-mono">Metrics: Waiting for pipeline activity...</p> |
| 1895 | | </div> |
| 1896 | | <div class="flex items-center space-x-3"> |
| 1897 | | <span class="text-xs font-mono bg-gray-800 border border-gray-700 rounded px-2.5 py-1 text-indigo-300">Target Profile: __MODEL_NAME_PLACEHOLDER__</span> |
| 1898 | | <button id="clearChatBtn" class="bg-red-900/40 hover:bg-red-800 border border-red-700 text-red-200 text-xs py-1.5 px-3 rounded transition-colors">Clear Chat History</button> |
| 1899 | | </div> |
| 1900 | | </header> |
| 1901 | | |
| 1902 | | <div class="grid grid-cols-1 lg:grid-cols-3 gap-6"> |
| 1903 | | <div class="lg:col-span-2 space-y-4"> |
| 1904 | | <div class="flex items-center space-x-4"> |
| 1905 | | <label class="font-medium text-sm">Select Stream Source:</label> |
| 1906 | | <select id="videoSelect" class="flex-1 bg-gray-800 border border-gray-700 rounded p-2 focus:outline-none focus:border-indigo-500"></select> |
| 1907 | | </div> |
| 1908 | | <div class="bg-black rounded-lg overflow-hidden aspect-video relative flex items-center justify-center border border-gray-800"> |
| 1909 | | <video id="videoPlayer" controls class="w-full h-full hidden"></video> |
| 1910 | | <div id="videoPlaceholder" class="text-gray-500 text-sm">No Active Video Stream Sample Loaded</div> |
| 1911 | | </div> |
| 1912 | | </div> |
| 1913 | | |
| 1914 | | <div class="flex flex-col h-[480px]"> |
| 1915 | | <div class="bg-gray-800 border border-gray-700 rounded-lg p-4 flex-1 flex flex-col min-h-0 relative overflow-hidden"> |
| 1916 | | <div class="flex justify-between items-center mb-3 flex-none"> |
| 1917 | | <h2 class="text-sm font-semibold tracking-wider text-gray-400 uppercase">Conversational History Log</h2> |
| 1918 | | <div id="busySpinner" class="hidden h-4 w-4 animate-spin rounded-full border-2 border-indigo-500 border-t-transparent"></div> |
| 1919 | | </div> |
| 1920 | | <div id="chatHistoryLog" class="flex-1 space-y-4 text-sm overflow-y-auto bg-gray-900 p-3 rounded border border-gray-750 font-mono min-h-0"> |
| 1921 | | <div class="text-gray-500 text-xs italic">System initialized. Awaiting prompt loop...</div> |
| 1922 | | </div> |
| 1923 | | </div> |
| 1924 | | |
| 1925 | | <div class="space-y-2 mt-4 flex-none"> |
| 1926 | | <textarea id="promptInput" rows="2" class="w-full bg-gray-800 border border-gray-700 rounded-lg p-3 text-sm focus:outline-none focus:border-indigo-500 resize-none placeholder-gray-500" placeholder="Ask a follow-up question..."></textarea> |
| 1927 | | <button id="submitBtn" class="w-full bg-indigo-600 hover:bg-indigo-500 disabled:bg-gray-700 disabled:cursor-not-allowed text-white font-medium py-2.5 px-4 rounded-lg transition-colors flex items-center justify-center space-x-2"> |
| 1928 | | <span id="btnText">Execute Analysis Prompt</span> |
| 1929 | | </button> |
| 1930 | | </div> |
| 1931 | | </div> |
| 1932 | | </div> |
| 1933 | | </div> |
| 1934 | | |
| 1935 | | <script> |
| 1936 | | const videoSelect = document.getElementById('videoSelect'); |
| 1937 | | const videoPlayer = document.getElementById('videoPlayer'); |
| 1938 | | const videoPlaceholder = document.getElementById('videoPlaceholder'); |
| 1939 | | const promptInput = document.getElementById('promptInput'); |
| 1940 | | const submitBtn = document.getElementById('submitBtn'); |
| 1941 | | const btnText = document.getElementById('btnText'); |
| 1942 | | const chatHistoryLog = document.getElementById('chatHistoryLog'); |
| 1943 | | const metricsPanel = document.getElementById('metricsPanel'); |
| 1944 | | const busySpinner = document.getElementById('busySpinner'); |
| 1945 | | const clearChatBtn = document.getElementById('clearChatBtn'); |
| 1946 | | |
| 1947 | | let chatHistoryBuffer = []; |
| 1948 | | |
| 1949 | | async function initializeApp() { |
| 1950 | | try { |
| 1951 | | const videoRes = await fetch('/api/videos'); |
| 1952 | | const videos = await videoRes.json(); |
| 1953 | | videos.forEach(v => videoSelect.add(new Option(v, v))); |
| 1954 | | |
| 1955 | | if(videos.length > 0) handleVideoChange(videos[0]); |
| 1956 | | } catch (e) { |
| 1957 | | chatHistoryLog.innerHTML = `<div class="text-red-400">Initialization Fault: ${e.message}</div>`; |
| 1958 | | } |
| 1959 | | } |
| 1960 | | |
| 1961 | | function appendMessageBlock(role, text, isSkeleton = false) { |
| 1962 | | const block = document.createElement('div'); |
| 1963 | | block.className = `p-2.5 rounded border ${role === 'user' ? 'bg-indigo-950/40 border-indigo-900/60 ml-6' : 'bg-gray-800/60 border-gray-700/50 mr-6'} ${isSkeleton ? 'skeleton-pulse min-h-[40px]' : ''}`; |
| 1964 | | |
| 1965 | | const senderLabel = document.createElement('div'); |
| 1966 | | senderLabel.className = `text-[10px] font-bold uppercase tracking-wider mb-1 ${role === 'user' ? 'text-indigo-400' : 'text-gray-400'}`; |
| 1967 | | senderLabel.textContent = role === 'user' ? '● User Prompt' : '● Model Response'; |
| 1968 | | |
| 1969 | | const contentText = document.createElement('div'); |
| 1970 | | contentText.className = "whitespace-pre-wrap leading-relaxed break-words text-sm font-mono text-gray-100"; |
| 1971 | | if (!isSkeleton) contentText.textContent = text; |
| 1972 | | |
| 1973 | | block.appendChild(senderLabel); |
| 1974 | | block.appendChild(contentText); |
| 1975 | | chatHistoryLog.appendChild(block); |
| 1976 | | chatHistoryLog.scrollTop = chatHistoryLog.scrollHeight; |
| 1977 | | return contentText; |
| 1978 | | } |
| 1979 | | |
| 1980 | | async function updateMetrics(clientLatencySec) { |
| 1981 | | try { |
| 1982 | | const res = await fetch('/api/metrics'); |
| 1983 | | const root = await res.json(); |
| 1984 | | const metrics = Object.values(root)[0]; |
| 1985 | | if (metrics) { |
| 1986 | | const tps = metrics.llm_average_token_per_second?.toFixed(1) || "0.0"; |
| 1987 | | const ttft = metrics.llm_first_infer_duration?.toFixed(2) || "0.00"; |
| 1988 | | const tokens = metrics.generated_token_num || 0; |
| 1989 | | metricsPanel.textContent = `Metrics: ${tps} tok/s • TTFT: ${ttft}s • ${tokens} tokens • Latency: ${clientLatencySec.toFixed(2)}s`; |
| 1990 | | } |
| 1991 | | } catch (e) { |
| 1992 | | console.error(e); |
| 1993 | | } |
| 1994 | | } |
| 1995 | | |
| 1996 | | function handleVideoChange(filename) { |
| 1997 | | resetChatHistory(); |
| 1998 | | if(!filename) { |
| 1999 | | videoPlayer.classList.add('hidden'); |
| 2000 | | videoPlaceholder.classList.remove('hidden'); |
| 2001 | | return; |
| 2002 | | } |
| 2003 | | videoPlaceholder.classList.add('hidden'); |
| 2004 | | videoPlayer.classList.remove('hidden'); |
| 2005 | | videoPlayer.src = `/stream/videos/${encodeURIComponent(filename)}`; |
| 2006 | | videoPlayer.load(); |
| 2007 | | } |
| 2008 | | |
| 2009 | | function resetChatHistory() { |
| 2010 | | chatHistoryBuffer = []; |
| 2011 | | chatHistoryLog.innerHTML = `<div class="text-gray-500 text-xs italic">Conversation wiped. Ready for prompt input...</div>`; |
| 2012 | | promptInput.value = "what is happening in this video?"; |
| 2013 | | } |
| 2014 | | |
| 2015 | | videoSelect.addEventListener('change', (e) => handleVideoChange(e.target.value)); |
| 2016 | | clearChatBtn.addEventListener('click', resetChatHistory); |
| 2017 | | |
| 2018 | | submitBtn.addEventListener('click', async () => { |
| 2019 | | const prompt = promptInput.value.trim(); |
| 2020 | | const video_name = videoSelect.value; |
| 2021 | | |
| 2022 | | if (!prompt || !video_name) return; |
| 2023 | | |
| 2024 | | const clientStartTime = performance.now(); |
| 2025 | | |
| 2026 | | appendMessageBlock('user', prompt); |
| 2027 | | chatHistoryBuffer.push({ role: 'user', content: prompt }); |
| 2028 | | |
| 2029 | | promptInput.value = ""; |
| 2030 | | submitBtn.disabled = true; |
| 2031 | | btnText.textContent = "Processing Inference..."; |
| 2032 | | busySpinner.classList.remove('hidden'); |
| 2033 | | |
| 2034 | | const liveResponseNode = appendMessageBlock('assistant', "Connecting...", true); |
| 2035 | | |
| 2036 | | try { |
| 2037 | | const response = await fetch('/api/analyze', { |
| 2038 | | method: 'POST', |
| 2039 | | headers: { 'Content-Type': 'application/json' }, |
| 2040 | | body: JSON.stringify({ video_name, history: chatHistoryBuffer }) |
| 2041 | | }); |
| 2042 | | |
| 2043 | | if (!response.ok) throw new Error("Server engine pipeline connection fault."); |
| 2044 | | |
| 2045 | | liveResponseNode.parentElement.classList.remove('skeleton-pulse'); |
| 2046 | | liveResponseNode.textContent = ""; |
| 2047 | | |
| 2048 | | const reader = response.body.getReader(); |
| 2049 | | const decoder = new TextDecoder(); |
| 2050 | | let buffer = ""; |
| 2051 | | let fullModelResponse = ""; |
| 2052 | | |
| 2053 | | while (true) { |
| 2054 | | const { value, done } = await reader.read(); |
| 2055 | | if (done) break; |
| 2056 | | |
| 2057 | | buffer += decoder.decode(value, { stream: true }); |
| 2058 | | const lines = buffer.split('\n'); |
| 2059 | | buffer = lines.pop(); |
| 2060 | | |
| 2061 | | for (const line of lines) { |
| 2062 | | const trimmed = line.trim(); |
| 2063 | | |
| 2064 | | if (!trimmed || !trimmed.startsWith('data: ')) continue; |
| 2065 | | |
| 2066 | | const dataStr = trimmed.slice(5).trim(); |
| 2067 | | if (dataStr === '[DONE]') continue; |
| 2068 | | |
| 2069 | | try { |
| 2070 | | const json = JSON.parse(dataStr); |
| 2071 | | if(json.error) { |
| 2072 | | liveResponseNode.textContent += `\n[AAF Error]: ${json.error}`; |
| 2073 | | continue; |
| 2074 | | } |
| 2075 | | |
| 2076 | | const contentToken = json.choices?.[0]?.delta?.content || ""; |
| 2077 | | if (contentToken) { |
| 2078 | | fullModelResponse += contentToken; |
| 2079 | | liveResponseNode.textContent = fullModelResponse; |
| 2080 | | chatHistoryLog.scrollTop = chatHistoryLog.scrollHeight; |
| 2081 | | } |
| 2082 | | } catch(e) {} |
| 2083 | | } |
| 2084 | | } |
| 2085 | | |
| 2086 | | chatHistoryBuffer.push({ role: 'assistant', content: fullModelResponse }); |
| 2087 | | |
| 2088 | | const clientLatencySec = (performance.now() - clientStartTime) / 1000; |
| 2089 | | setTimeout(() => updateMetrics(clientLatencySec), 500); |
| 2090 | | |
| 2091 | | } catch (err) { |
| 2092 | | liveResponseNode.parentElement.classList.remove('skeleton-pulse'); |
| 2093 | | liveResponseNode.textContent = `\n[Pipeline Runtime Exception]: ${err.message}`; |
| 2094 | | } finally { |
| 2095 | | submitBtn.disabled = false; |
| 2096 | | btnText.textContent = "Execute Analysis Prompt"; |
| 2097 | | busySpinner.classList.add('hidden'); |
| 2098 | | } |
| 2099 | | }); |
| 2100 | | |
| 2101 | | initializeApp(); |
| 2102 | | </script> |
| 2103 | | </body> |
| 2104 | | </html> |
| 2105 | | """ |
| 2106 | | return HTMLResponse(content=html_content.replace("__MODEL_NAME_PLACEHOLDER__", TARGET_MODEL)) |
| 2107 | | |
| 2108 | | if __name__ == "__main__": |
| 2109 | | uvicorn.run(app, host=args.host, port=args.port) |
| 2110 | | EOF |
| 2111 | | }}} |
| 2112 | | - create a python virtual env (always a good idea to keep python dependencies containerized) and install python modules we need: |
| 2113 | | {{{#!bash |
| | 665 | Requirements: |
| | 666 | - Ara runtime |
| | 667 | - eIQ AAF Connector |
| | 668 | - Qwen2.5-VL-7B-Instruct-Ara240 model |
| | 669 | |
| | 670 | Steps: |
| | 671 | 1. create a python virtual env (always a good idea to keep python dependencies containerized) and install python libs we need: |
| | 672 | {{{#!bash |
| | 673 | # create a dir for the venv |
| | 674 | mkdir webvlm |
| | 675 | cd webvlm |
