Large Data Transfers

How to transfer multi-megabyte waveforms and spectra from instruments with PyVISA. Binary protocols, chunked reads, and memory-efficient patterns.

Modern oscilloscopes produce 10M+ sample waveforms (40+ MB). Standard PyVISA settings can be 10-100x slower than needed. Binary format and proper chunk sizes fix most of it.

Binary vs ASCII Comparison

import pyvisa
import time
import numpy as np

rm = pyvisa.ResourceManager()
scope = rm.open_resource("USB0::0x0699::0x0363::C102912::INSTR")
scope.timeout = 30000

# ASCII: human-readable, slow
start = time.time()
scope.write("FORM:DATA ASC")
ascii_data = scope.query("CURV?")
ascii_values = [float(x) for x in ascii_data.split(",")]
ascii_time = time.time() - start

# Binary: raw bytes, fast
start = time.time()
scope.write("FORM:DATA REAL,32")
binary_values = scope.query_binary_values("CURV?", datatype="f")
binary_time = time.time() - start

print(f"ASCII:  {ascii_time:.2f}s ({len(ascii_values)} points)")
print(f"Binary: {binary_time:.2f}s ({len(binary_values)} points)")
print(f"Speedup: {ascii_time / binary_time:.1f}x")

# Typical result for 1M points:
# ASCII:  12.3s
# Binary: 0.8s
# Speedup: 15.4x

scope.close()
rm.close()

Log measurements automatically

TofuPilot records test results from your PyVISA scripts, tracks pass/fail rates, and generates compliance reports. Free to start.

Fast Waveform Capture

Set binary encoding, increase chunk_size, and use *OPC? to wait for acquisition.

import pyvisa
import numpy as np
import time

rm = pyvisa.ResourceManager()
scope = rm.open_resource("USB0::0x0699::0x0363::C102912::INSTR")
scope.timeout = 30000
scope.chunk_size = 1024 * 1024  # 1 MB chunks

scope.write("*RST")
scope.write("*CLS")

# Binary format, 16-bit samples
scope.write("DATA:ENC RIBINARY")
scope.write("DATA:WIDTH 2")
scope.write("DATA:START 1")
scope.write("DATA:STOP 10000000")  # Up to 10M points

# Fastest acquisition mode
scope.write("ACQ:MODE SAMPLE")
scope.write("HOR:RECORDLENGTH 10000000")
scope.write("ACQ:STOPAFTER RUNSTOP")
scope.write("ACQ:STATE ON")

scope.query("*OPC?")

# Read scaling from preamble
scope.write("DATA:SOURCE CH1")
preamble = scope.query("WFMPRE?").split(",")
y_scale = float(preamble[13])
y_offset = float(preamble[14])
x_scale = float(preamble[9])

# Transfer
start = time.time()
raw = scope.query_binary_values("CURVE?", datatype="h")
elapsed = time.time() - start

voltages = np.array(raw) * y_scale + y_offset
time_axis = np.arange(len(voltages)) * x_scale

print(f"{len(voltages)} points in {elapsed:.2f}s")
print(f"Transfer rate: {len(voltages) / elapsed / 1e6:.1f} MSa/s")

scope.close()
rm.close()

Chunked Transfer for Very Large Waveforms

When a waveform exceeds available memory (or you want progress feedback), transfer it in chunks.

import pyvisa
import numpy as np

def chunked_waveform(instrument, total_points, chunk_size=1000000):
    """Transfer a large waveform in chunks."""
    all_data = []

    for start in range(1, total_points + 1, chunk_size):
        end = min(start + chunk_size - 1, total_points)

        instrument.write(f"DATA:START {start}")
        instrument.write(f"DATA:STOP {end}")

        chunk = instrument.query_binary_values("CURVE?", datatype="h")
        all_data.extend(chunk)

        done = len(all_data)
        print(f"{done}/{total_points} points ({100 * done / total_points:.0f}%)")

    return np.array(all_data)

# Usage
rm = pyvisa.ResourceManager()
scope = rm.open_resource("USB0::0x0699::0x0363::C102912::INSTR")
scope.timeout = 60000
scope.chunk_size = 1024 * 1024

scope.write("DATA:ENC RIBINARY")
scope.write("DATA:WIDTH 2")

waveform = chunked_waveform(scope, total_points=50000000)
print(f"Total: {len(waveform)} points")

scope.close()
rm.close()

Spectrum Analyzer Trace

Binary format works the same way for spectrum analyzers.

import pyvisa
import numpy as np
import time

rm = pyvisa.ResourceManager()
sa = rm.open_resource("TCPIP::192.168.1.101::INSTR")
sa.timeout = 60000

# Binary format, single sweep
sa.write("FORM:DATA REAL,32")
sa.write("INIT:CONT OFF")
sa.write("SWE:POIN 50001")
sa.write("FREQ:START 1e6")
sa.write("FREQ:STOP 6e9")

sa.write("INIT:IMM")
sa.query("*OPC?")

start = time.time()
trace = sa.query_binary_values("TRAC? TRACE1", datatype="f")
elapsed = time.time() - start

freqs = np.linspace(1e6, 6e9, len(trace))
print(f"{len(trace)} points in {elapsed:.2f}s")

sa.close()
rm.close()

Streaming FFT (Chunked Processing)

For waveforms too large to FFT in one shot, process power spectra per chunk and average.

import numpy as np
import gc

def streaming_fft(instrument, chunk_size=1000000):
    """Compute averaged power spectrum from a large waveform in chunks."""
    total = int(instrument.query("HOR:RECORDLENGTH?"))
    sample_rate = float(instrument.query("HOR:SAMPLERATE?"))

    accumulator = None
    n_chunks = 0

    for start in range(1, total + 1, chunk_size):
        end = min(start + chunk_size - 1, total)
        instrument.write(f"DATA:START {start}")
        instrument.write(f"DATA:STOP {end}")

        chunk = instrument.query_binary_values("CURVE?", datatype="h")
        arr = np.array(chunk, dtype=np.float32)

        power = np.abs(np.fft.fft(arr)) ** 2
        if accumulator is None:
            accumulator = power
        else:
            accumulator += power

        n_chunks += 1
        del chunk, arr, power
        gc.collect()
        print(f"Chunk {n_chunks} processed")

    avg_power = accumulator / n_chunks
    freqs = np.fft.fftfreq(len(avg_power), 1 / sample_rate)

    # Return positive half
    half = len(freqs) // 2
    return freqs[:half], avg_power[:half]

TCP/IP Tuning

For Ethernet instruments, connection parameters affect throughput.

import pyvisa

rm = pyvisa.ResourceManager()

# VXI-11 (standard)
inst = rm.open_resource(
    "TCPIP::192.168.1.100::inst0::INSTR",
    timeout=60000,
    chunk_size=2 * 1024 * 1024,  # 2 MB
)

# Raw socket (lower latency, if instrument supports port 5025)
# inst = rm.open_resource(
#     "TCPIP::192.168.1.100::5025::SOCKET",
#     read_termination="\n",
#     write_termination="\n",
#     timeout=60000,
#     chunk_size=2 * 1024 * 1024,
# )

print(inst.query("*IDN?").strip())

Avoid WiFi for large transfers

Wired Ethernet gives 10-100x more consistent throughput. WiFi latency spikes cause timeout errors on multi-megabyte transfers.

Configuration Checklist

Setting	Default	Recommended	Why
Data format	ASCII	`RIBINARY` or `REAL,32`	10-50x faster
`chunk_size`	20 KB	1-2 MB	Fewer read calls
`timeout`	2000 ms	30000-60000 ms	Large transfers take time
Data width	Varies	2 (16-bit)	Good resolution, half the size of 32-bit
Header	ON	`HEADER OFF`	Simplifies binary parsing
Connection	Open/close per test	Keep open	Saves 1-5 s per connection