Second milestone

For the second milestone I got done simple answering engine. Gesture "analysis" is virtually non-existing at the moment, but the system will try to answer with phrases that are similar length to the actually played. Phrases are chosen randomly from all recorded.

Here's the code:

(
var numinchannels, samplerate, controlrate, masterVol, bufferArray, longbufferLength, fftbuflength, muteInput, in, out, st_out, timeOffset, pathprefix, inputBoost, oldstarttime, minimalEventDuration, currentEventNumber, onsetOffset; //general variables
var recorder, play, straightPlayer, warpPlayer, rec1, time, playCnt, checkPlayers, checkDuration, phrasePlay, clear, getPhrases, phraseAnalyze, phraseRespond; //synthdefs and functions
var longbuf, fftbuf; //buffers
var durationTriggerValue, dursMult, dursMultRand, stimes, stimesRand, timesEnv, timesRand, rates, ratesRand, pitches, pitchesRand, az, azRand, az2diff, el, elRand, el2diff, modAmts, modAmtsRand, modFreqs, modFreqsRand, atts, attsRand, rels, relsRand, amps, ampsRand; //envelopes
var window1, midiNoteKnob, midiAvKnob, hasFreqAvKnob, ampAvKnob, loudnessAvKnob, amplitudeKnob, starttimeNumber, durationNumber, recButton, playButton, onsetNumber, playNumber, clearButton, phraseNumber; //gui
var startTime, stopTime, midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, amplitude, duration; //continous values
var mainList, phraseList, durPhrases, pitchPhrases; //arrays with events and phrases


// ----------- variables to set ------------------------- //
in = 0; //number of the input channel
out = 42; //number of the first channel of Ambisonics B-format output
st_out = 0; //number of the first channel of stereo out
masterVol = 0; //in dB
inputBoost = 0; //input amplification in dB
minimalEventDuration = 0.2; //in seconds - to avoid unnecessary retriggering
longbufferLength = 90; //length of rec buffers in seconds
onsetOffset = -0.08; //compensate for onset delay; negative values if onsets are too late; -0.08 seems to be fine at the moment


// INIT


numinchannels = 1; //currently only one input channel is supported
inputBoost = inputBoost.dbamp; //convert to amp
currentEventNumber = 0; //init event numbering
startTime = 0; //init start time
samplerate = Server.default.sampleRate; //get samplerate
controlrate = samplerate / Server.default.options.blockSize; //get control rate
if (samplerate < 50000, {fftbuflength = 1024}, {if (samplerate < 100000, {fftbuflength = 2048}, {fftbuflength = 4096})}); // set the FFT buffer size // fftbuflength.postln; longbuf = CtkBuffer.buffer(size: samplerate * longbufferLength).load(sync: true); //main recording buffer mainList = Array.newClear(7) ! 1000; //set up the global event list for the buffer; values for each array inside: startTime, midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, duration phraseList = Array.newClear(7) ! 1000; //list of detected phrases //durPhrases = Array.new; //list of phrases with similar dur //pitchPhrases = Array.new; //list of phrases with similar pitch timeOffset = thisThread.seconds; // get runtime time // SYNTHDEFS recorder = CtkSynthDef(\recorder, {arg mute = 0, onsetTreshold = 0.8, treshHi = -36, treshLo = -52, averageDelay = 0.1, relaxtime = 0.1, floor = -30; //tresholds to be adjusted var recTrigger, stopTrigger, amplitude, timer, hasFreq, freq, trigger, freqSlope, sig, hasFreqAv, ampAv, freqAv, counter, recTrigger0, inGate, chain, localbuf, onsets, midiNote, midiSlope, midiAv, loudness, loudnessAv; inGate = Gate.ar(SoundIn.ar(in) * inputBoost, mute.neg + 1) * (mute.neg + 1); //incoming sound with mute amplitude = Amplitude.kr(LagUD.ar(inGate, 0.01, 0.4), 0.01, 1.8); //amplitude tracking with "smoothing" trigger = Schmidt.kr(amplitude, treshLo.dbamp, treshHi.dbamp); //general trigger sig = inGate;//Gate.ar(inGate, trigger); //not gated for now localbuf = LocalBuf.new(fftbuflength, 1); //fft buffer chain = FFT(localbuf, SoundIn.ar(in), 0.5); //start the chain onsets = Gate.kr(Onsets.kr(chain, onsetTreshold, relaxtime: relaxtime, floor: floor.dbamp), trigger) * trigger; //onset detection, gated and automaticaly "muted" loudness = Loudness.kr(chain); //loudness analysis // stopTrigger = onsets.neg + 1; RecordBuf.ar(sig, longbuf, 0, run: 1, loop: 0, trigger: 1); //recorder #freq, hasFreq = Tartini.kr(sig); //pitch analysis midiNote = Gate.kr(freq.cpsmidi, trigger); //convert to midi notes // freqSlope = Slope.kr(freq).abs; //kind of amount of change of freq midiSlope = (midiNote - Delay1.kr(midiNote)).abs; //kind of amount of change of midi note midiAv = Gate.kr(AverageOutput.kr(midiSlope, DelayC.kr(onsets, 2, averageDelay)), trigger); //average of midi note change // freqAv = Gate.kr(AverageOutput.kr(freqSlope, DelayC.kr(onsets, 2, averageDelay)), trigger); //average base freq in the sample hasFreqAv = Gate.kr(AverageOutput.kr(hasFreq, DelayC.kr(onsets, 2, averageDelay)), trigger); //average amount of pitched sounds in the sample ampAv = AverageOutput.kr(amplitude, DelayC.kr(onsets, 2, averageDelay)); //average amplitude of the sample loudnessAv = AverageOutput.kr(loudness, DelayC.kr(onsets, 2, averageDelay)); //average lodness timer = Phasor.kr(1, controlrate.reciprocal, 0, longbufferLength); //timer to report rec time SendReply.kr(onsets, 'start' , [timer]); //send start times for events SendReply.kr(Impulse.kr(10), 'cont' , [midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, amplitude]); //send continous values for events SendReply.kr(trigger.neg + 1, 'stop' , [timer]); //send stop times for events // counter.poll(1, label: \counter); // trigger.poll(1, label: \trigger); // env.poll(1, label: \env); // hasFreqAv.poll(1, label: \hasFreqAv); // stopTrigger.poll(1, label: \stop); // recTrigger4.poll(1, label: \recTrigger4); // amplitude.ampdb.poll(1, label: \amp); // SendTrig.kr(recTrigger, 0, 1); // SendTrig.kr(stopTrigger, 0, 0); // SendTrig.kr(stopTrigger, 1, timer); }); straightPlayer = CtkSynthDef(\straightPlayer, {arg dur, buffer = longbuf, startPos, amp = 0, rate = 1, az = 0, el = 0, dir=1, modAmt = 0, modFreq = 0, att = 0.001, rel = 0.001; var src, env, enc_w, enc_x, enc_y, enc_z, decoded, mod, secondLeg; secondLeg = dur - (dur * att) - (dur * rel); env = EnvGen.kr(Env([0.001, 1, 1, 0.001], [att * dur, secondLeg, rel * dur], \exponential)); az = (az * pi) / 180; //to use az in degrees el = (el * pi) / 180; //to use el in degrees mod = SinOsc.ar(modFreq, modAmt); // src = PlayBuf.ar(numinchannels, buffer, (rate + mod) * BufRateScale.kr(buffer), 1, startPos * numinchannels * samplerate, 0) * env; src = PlayBuf.ar(numinchannels, buffer, rate * BufRateScale.kr(buffer), 1, startPos * samplerate, 0) * env; // src = PlayBuf.ar(1, buffer); # enc_w, enc_x, enc_y, enc_z = BFEncode1.ar(src, az, el, 1, 1, 0); Out.ar(st_out, B2UHJ.ar(enc_w, enc_x, enc_y, enc_z) * amp.dbamp * masterVol.dbamp); //stereo out // Out.ar(out, [enc_w, enc_x, enc_y, enc_z] * amp.dbamp * masterVol.dbamp); //ambisonics encoded out }); //warpPlayer = CtkSynthDef(\warpPlayer, {arg dur, buffer, amp = 0, length, rate = 1, pitch = 1, az = 0, el = 0, dir=1, modAmt = 0, modFreq = 0, att = 0.001, rel = 0.001; //AM!!! // var src, pointer, env, enc_w, enc_x, enc_y, enc_z, decoded, mod, secondLeg; // secondLeg = dur - (dur * att) - (dur * rel); // env = EnvGen.kr(Env([0.001, 1, 1, 0.001], [att * dur, secondLeg, rel * dur], \exponential)); //// pointer = EnvGen.kr(Env([0, length / bufferslength], [rate * length / bufferslength])); //// pointer = RedPhasor2.ar(Impulse.kr(0), rate / samplerate, 0, length / bufferslength, 1); // mod = SinOsc.ar(modFreq, modAmt) + 1; // pointer = Phasor.ar(Impulse.kr(0), rate * bufferslength / samplerate, 0, length / bufferslength); //// pointer.poll(2, label: \pointer); // az = (az * pi) / 180; //to use az in degrees // el = (el * pi) / 180; //to use el in degrees // src = Warp1.ar(numinchannels, buffer, pointer, pitch, 0.12, -1, 4, 0.2, 4) * env * mod; // # enc_w, enc_x, enc_y, enc_z = BFEncode1.ar(src, az, el, 1, 1, 0); // Out.ar(st_out, B2UHJ.ar(enc_w, enc_x, enc_y, enc_z) * amp.dbamp * masterVol.dbamp); //stereo out //// Out.ar(out, [enc_w, enc_x, enc_y, enc_z] * amp.dbamp * masterVol.dbamp); //ambisonics encoded out // }); // RECORDER CONTROL rec1 = recorder.new; //initialize recorder rec1.play; //start recorder // FUNCTIONS time = {thisThread.seconds - timeOffset}; //get time relative to start checkDuration = {arg type; //resonse player inlined //type: 0 - stopTime, 1 = startTime if(type==0, {duration = stopTime - startTime; mainList.at(currentEventNumber).putEach(6, duration); if(durPhrases.size>0, {phrasePlay.value(durPhrases.at(durPhrases.size.rand))}); if(mainList[currentEventNumber][0] == nil, {mainList.at(currentEventNumber).putEach(0, startTime)}); currentEventNumber = currentEventNumber + 1; {durationNumber.value = duration; recButton.value = 0}.defer},
{if(stopTime < oldstarttime, {duration = startTime - oldstarttime; // 'oldstarttime: '.post; oldstarttime.postln; // 'startTime: '.post; startTime.postln; // 'duration: '.post; duration.postln; mainList.at(currentEventNumber).putEach(6, duration); if(durPhrases.size>0, {phrasePlay.value(durPhrases.at(durPhrases.size.rand))}); if(mainList[currentEventNumber][0] == nil, {mainList.at(currentEventNumber).putEach(0, startTime)}); currentEventNumber = currentEventNumber + 1; {durationNumber.value = duration; recButton.value = 0}.defer})}
);
// getPhrases.value;
// phraseAnalyze;
phraseRespond.value;
};


phrasePlay = {arg phraseNo;
var pos, dur;
'phrase no: '.post; phraseNo.postln;
pos = mainList[phraseNo][0];
dur = mainList[phraseNo][6];
'startPos: '.post; pos.post; ', duration: '.post; dur.postln;
straightPlayer.new(0, dur).dur_(dur).startPos_(pos).play;
};


clear = { // free whatever possible = stop execution
rec1.free;
longbuf.free;
AppClock.sched(0.53, {window1.close});
// Server.default.quit;
// Server.default.boot;
};


//getPhrases = { //startTime, midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, duration
// var olddur = 0, oldmidi = 0, durvartrigger = 0.2, thisPhrase = 0;
// (currentEventNumber - 1).do({arg i;
// var dur, midi, pStarttime, pStoptime;
// dur = mainList.at(i).at(6);
// dur =
// if(((1-durvartrigger)*dur) < olddur && ((1+durvartrigger)*dur) > olddur, phraseList.at(thisPhrase).putEach(0, duration)
// )
//
// });
// };


//phraseAnalyze = {arg lookBackTime = 3;
// var phraseNo = 0, phraseTime;
// while
// }


phraseRespond = { //as far as have nils in current phrase...
//find similar phrases and play them...
durPhrases = Array.new; //list of phrases with similar dur
pitchPhrases = Array.new; //list of phrases with similar pitch
(currentEventNumber - 1).do({arg i;
// "dur from phraseRespond: ".post; mainList.at(i).at(6).postln;
if((mainList.at(i).at(6) / mainList.at(currentEventNumber-1).at(6)).abs > 0.4, {durPhrases = durPhrases.add(i)}); //find similar note durations
durPhrases.postln;
durPhrases.size.rand.postln;
durPhrases.at(durPhrases.size.rand).postln;
// if(durPhrases.size>0, {phrasePlay.value(durPhrases.at(durPhrases.size.rand))}); //too many triggered !!!
});
};


//var startTime, stopTime, midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, amplitude, duration;


// ------ a simple GUI --------


window1 = Window.new('Improviser parameters').front;
window1.view.decorator=FlowLayout(window1.view.bounds);
midiNoteKnob = EZKnob(window1, 50@90,"midiNote", [0, 127]);
midiAvKnob = EZKnob(window1, 50@90,"midiSlAv", [0, 0.2]);
hasFreqAvKnob = EZKnob(window1, 50@90,"hasFreq", [0, 1]);
ampAvKnob = EZKnob(window1, 50@90,"ampAv", [0, 1]);
loudnessAvKnob = EZKnob(window1, 50@90,"loudAv", [0, 80]);
amplitudeKnob = EZKnob(window1, 50@90," amp ", [0, 1]);
recButton = Button(window1, Rect(20,20,50,30)).states_([["waiting", Color.gray, Color.black], ["phrase", Color.black, Color.red]]);
starttimeNumber = EZNumber(window1, 150@20,"s_time (s)", [0, longbufferLength]);
durationNumber = EZNumber(window1, 150@20,"dur (s)", [0, longbufferLength]);
onsetNumber = EZNumber(window1, 150@20,"onset no", [0, 10000]);
// phraseNumber = EZNumber(window1, 150@20,"phrase no", [0, 10000]);
playNumber = EZNumber(window1, 150@20,"play phrase no", [0, 10000], {|ez| phrasePlay.value(ez.value)}, labelWidth: 120);
clearButton = Button(window1, Rect(20,20,70,30)).states_([["kill me", Color.gray, Color.black], ["AAAHHH!!!", Color.red, Color.black]]).action_({ arg butt; clear.value});


// RESPONDERS
OSCresponder(nil, 'start', { arg t, r, msg;// msg.postln;
oldstarttime = startTime; //remember previous start time
startTime = msg.at(3) + onsetOffset; //get actual start time
checkDuration.value(1); //check if we need to take duration from it
mainList.at(currentEventNumber).putEach(0, startTime); //put start time into the current phrase array
// 'starttime: '.post; startTime.postln;
{recButton.value = 1;
starttimeNumber.value = startTime;
onsetNumber.value = currentEventNumber;
}.defer; //gui control
}).add;


OSCresponder(nil, 'stop', { arg t, r, msg;// msg.postln;
// var stoptime;
stopTime = msg.at(3); //get stop time
checkDuration.value(0); //put duration in the array
// 'stoptime: '.post; stopTime.postln;
// {recButton.value = 0;
// durationNumber.value = duration;
// }.defer; //gui contol
}).add;


OSCresponder(nil, 'cont', { arg t, r, msg;// msg.postln;
var freqAv, hasFreqAv, ampAv, msgNew;
// [midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, amplitude]
{
midiNoteKnob.value = msg.at(3);
midiAvKnob.value = msg.at(4);
hasFreqAvKnob.value = msg.at(5);
ampAvKnob.value = msg.at(6);
loudnessAvKnob.value = msg.at(7);
amplitudeKnob.value = msg.at(8);
}.defer; //gui control
// 'org array: '.post; msg.postln;
msgNew = msg.rotate(-3).clipExtend(5); //extract only useful values from the array
// 'new array: '.post; msgNew.postln;
msgNew.do{arg val, i; mainList.at(currentEventNumber).putEach(i+1, val)};//put parameters into the main array
// 'current event no: '.post; currentEventNumber.postln;
}).add;
)