Creating the waveform frame

What we are trying to accomplish here is to create a simple and yet smooth waveform visualizer for the song we play with our basic player extension (the AudioPlayer class we made earlier), but that can really work for every discrete signal in input. The basic idea is that the frame is updated each time a new array of bytes (pcmdata, the signal) is processed and send using a function we'll call updateWave(). What are we gonna do is:

• Get the current signal, taking the pcmdata in input
• Create a N_TASK of tasks to work outside the swing thread
• Each task will make the math and calculate the position of screen of the polyline for a part of the signal
• Once all the signal is calculated we call repaint() to make awt update the panel.

What are we doing in theory is called result parallelism, we'll crate more task that works in parallel (sorta, depends by the number of processors of the machine) to solve our problem. We'll have to coordinate the task by using a taskCount variable and a synchronized block (we don't want lost updates), when taskCount reach N_TASK, we know we have to call repaint(). Now, let's see the code.

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266

package it.pievis.GUI; import it.pievis.utils.BackgroundExecutor; import it.pievis.utils.BarrierMonitorLock; import it.pievis.utils.Timer; import it.pievis.utils.Utils; import java.awt.Graphics; import java.awt.Graphics2D; import java.awt.Label; import java.awt.RenderingHints; import java.util.concurrent.Executor; import javax.swing.ImageIcon; import javax.swing.JFrame; import javax.swing.JPanel; public class WaveformParallelFrame extends JFrame { private int WIDTH = 450; private int HEIGHT = 100; private int N_TASK = 2; private int NTOTLINES = 0; private int DISCARD_FACTOR = 4; //must be %2=0 ( < 8 ) private WaveformPanel wfp; private int[] polyliney, polylinex; //contains positions for the polyline //Executor/Other Executor executor = BackgroundExecutor.get(); boolean updatedOnScreen = true; private boolean canWriteOnScreen = false; //Icons ImageIcon frameIcon = new ImageIcon(getClass().getResource("/res/waveicon.png")); //Check correctness/performance int taskCount = 0; Timer timer = new Timer(); //timer for max/min drawtime public WaveformParallelFrame() { super(); setIconImage(frameIcon.getImage()); setSize(WIDTH, HEIGHT+20); setTitle("Waveform Frame"); setName("Main FRAME"); wfp = new WaveformPanel(); wfp.setSize(WIDTH, HEIGHT); wfp.setName("Waveform PANEL"); //wfp.setDoubleBuffered(false); add(wfp); } public void updateWave(byte[] pcmdata) { wfp.updateWave(pcmdata); } /** * This panel gets a signal with updateWave and each time * he can (not busy), he asks the executor to run * N_TASK tasks that calculate the absolute position of the * polyline relative to part of the signal (that is pcmdata) * @author Pierluigi */ class WaveformPanel extends JPanel { byte[] pcmdata = null; Label cdtlmx; //Label per il drawtime massimo Label cdtlmn; //Label per il drawtime minimo Label cdtlavg; //Label per il drawtime medio public WaveformPanel() { super(); setLayout(null); cdtlmx = new Label("DrawTime max"); cdtlmx.setBounds(0, 0, 80, 10); add(cdtlmx); cdtlmn = new Label("DrawTime min"); cdtlmn.setBounds(0, 10, 80, 10); add(cdtlmn); cdtlavg = new Label("DrawTime avg"); cdtlavg.setBounds(160, 0, 80, 10); add(cdtlavg); } /** * Refresh the wave every times a new pcmdata arrives */ public void updateWave(byte[] pcmdata) { //ignore all other pcmdata until we draw something //repaint(); synchronized (wfp) { if(!updatedOnScreen) return; updatedOnScreen = false; } this.pcmdata = pcmdata; callTask(); } /** * This makes the executor run some task * each task calculate position for part of the signal */ private void callTask() { timer.start(); int numLines = pcmdata.length/4; // half because 2 points = 1 line, other half because we use 16 bit samples numLines/=DISCARD_FACTOR; //Discard other lines for performance (no quality but speed). //Instantiate the array if the number of total lines changes //This might happen due to different pcm lenght from song to song if(NTOTLINES != numLines){ NTOTLINES = numLines; instantiateEmptyLinesArray(); log("Lines we are drawing: " + numLines ); } //Let multiple task do the math int diff = pcmdata.length / N_TASK; for(int i = 0; i<N_TASK; i++) executor.execute(new WaveformTask(getWidth(), getHeight(), i*diff, (i+1)*diff, i)); } /** * Handle the refresh of the waveform * @param g */ private void doDrawing(Graphics g){ Graphics2D g2d = (Graphics2D) g; g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_OFF); if(pcmdata == null){ int HEIGHT = getHeight(); int WIDTH = getWidth(); //Render a straight line g2d.drawLine(0, HEIGHT/2, WIDTH, HEIGHT/2); return; } //Let swing handle the drawing if(polylinex != null){ drawLines(g2d); } } /** * This task calculates part of the polyline, relative to a portion * of the signal (pcmdata.lenght/N_TASK) * @author Pierluigi */ class WaveformTask implements Runnable { int HEIGHT; int WIDTH; int from; int to; int N; public WaveformTask(int width, int height, int from, int to, int n) { HEIGHT = height; WIDTH = width; this.to = to; this.from = from; this.N = n; } @Override public void run() { //log("Task " + N + " inizia l'esecuzione"); calcLine2d(); //The last thread synch with the drawing arrays //log("Task " + N + " ha completato l'esecuzione"); synchronized (polylinex) { taskCount++; if(taskCount == N_TASK){ taskCount = 0; canWriteOnScreen = true; repaint(); //If I'm the last one, then repaint } } } void calcLine2d(){ float scale = (float) HEIGHT/65536; //h/2^16 int npoints = (to-from)/(2*DISCARD_FACTOR); //16bit, half gone //log( "from: " + from + " to: " + to); float pwidth = (float) WIDTH/N_TASK; float dx = (pwidth)*N; int dy = HEIGHT/2; float lineLen = (float) pwidth/npoints; int ix = 0; //relative x position int absi; //absolute index of the arrays int inc = DISCARD_FACTOR * 2; for(int i = from; i < to; i+=inc){ int sample0 = Utils.getSixteenBitSample(pcmdata[i+1], pcmdata[i]); int val0 = (int) (sample0*scale)+dy; int diffx0 = Math.round(lineLen*ix+dx); absi = ix+(N*npoints); WaveformParallelFrame.this.polylinex[absi] = diffx0; WaveformParallelFrame.this.polyliney[absi] = val0; ix++; //log("x vals: " + diffx0 + " --" + nlines + " from: " + from + " to: " + to+ " DX: " + dx); //log("Updated GUI ( " + sumData + ") " + lineLen + " " + WIDTH + " " + HEIGHT + " nlines: " +nlines + " Scale: "+scale ); } } } //TASK DEFINITION END /** * This should draw lines * @param g2d */ void drawLines(Graphics2D g2d) { assert(polylinex != null); //Was everything instantiated with success ? assert(taskCount == 0); //Have all the task processed their wave for the cycle? if(canWriteOnScreen){ //repaint() might be called from something else (window resize, etc) //log("Inizio a disegnare..."); g2d.drawPolyline(polylinex, polyliney, polylinex.length); g2d.dispose(); timer.stop(); //log("Disegno eseguito."); synchronized (wfp) { canWriteOnScreen = false; updatedOnScreen = true; //sync with pcmdata input } } } /** * Called each time the UI is rendered */ @Override protected void paintComponent(Graphics g) { super.paintComponent(g); doDrawing(g); cdtlmx.setText(timer.getMax() + ""); cdtlmn.setText(timer.getMin() + ""); cdtlavg.setText(timer.getAvg() + ""); } /** * Initialize arrays */ private void instantiateEmptyLinesArray() { polylinex = new int[NTOTLINES*2]; polyliney = new int[NTOTLINES*2]; for(int i = 0; i<NTOTLINES*2; i++) { polylinex[i] = 0; polyliney[i] = 0; } } } /// END OF JPANEL CLASS private void log(String line) { System.out.println("WF out] " + line); } }

I hope the code is enough to understand how to draw a waveform in a multitask fashion way. As you can see I had to cut some data out for performance issue, that's because in this case we want a fast response, if we wanted to do it for the signal to be more precise I would had to draw every 16bit sample of the pcmdata.

Implementing UI callbacks with audio player functionality

In the latest posts we discussed how to create the User Interface of our application and how to wrap all the player audio functionality into one simple AudioPlayer class. Now we are gonna put the two things together, the finishing result should be something that slightly works.

This is the uiBehaviour() procedure we left behind (MainView class).

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242

private void uiBehaviour() { //File chooser fc.setMultiSelectionEnabled(true); fc.setFileFilter(new FileFilter() { @Override public String getDescription() { return "only supported audio files (mp3, wav)"; } @Override public boolean accept(File f) { if(f.isDirectory()) return true; if(f.getName().endsWith(".mp3")) return true; if(f.getName().endsWith(".wav")) return true; return false; } }); btnAdd.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { int returnVal = fc.showOpenDialog(btnAdd); if(returnVal == JFileChooser.APPROVE_OPTION){ File[] files = fc.getSelectedFiles(); for(File f : files){ player.addSong(f.getAbsolutePath()); songList.addElement(f.getName()); log("Added file " + f.getName() + " to playlist"); } } else{ log("No file selected"); } } }); //Song List jSongList.setModel(songList); jSongList.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); jSongList.setLayoutOrientation(JList.VERTICAL); //Event that triggers at double click jSongList.addMouseListener(new MouseAdapter() { public void mouseClicked(MouseEvent evt){ JList list = (JList)evt.getSource(); if(evt.getClickCount() == 2){ log("Double click detected, moving to selected item."); int index = list.locationToIndex(evt.getPoint()); player.setIndexSong(index); try { player.play(); } catch (BasicPlayerException ev) { ev.printStackTrace(); } } } }); //Btn Delete btnDel.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { //Executed Outside UI Thread BackgroundExecutor.get().execute(new Runnable() { @Override public void run() { int[] indexes = jSongList.getSelectedIndices(); int removed = 0; for(int i : indexes) { log("Removed Song ("+(i-removed)+")" + songList.get(i-removed)); player.removeSong(i-removed); songList.remove(i-removed); removed++; } } }); } }); //Play Btn btnPlay.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent e) { try { tooglePlay(); } catch (BasicPlayerException e1) { e1.printStackTrace(); } } }); //Next and Previous btns btnNext.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent arg0) { try { player.nextSong(); //seekbar.resetLastSeek(); } catch (BasicPlayerException e) { log("Error calling the next song"); e.printStackTrace(); } } }); btnPrev.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent arg0) { try { player.prvSong(); //seekbar.resetLastSeek(); } catch (BasicPlayerException e) { log("Error calling the previous song"); e.printStackTrace(); } } }); //Player related behaviour player.addBasicPlayerListener(new BasicPlayerListener() { @Override public void stateUpdated(BasicPlayerEvent event) { if(event.getCode() == BasicPlayerEvent.EOM) { //seekbar.resetLastSeek(); try { player.nextSong(); } catch (BasicPlayerException e) { e.printStackTrace(); } log("EOM event catched, calling next song."); } if(event.getCode() == BasicPlayerEvent.PAUSED){ //btnPlay.setText(">"); btnPlay.setIcon(playIcon); } if(event.getCode() == BasicPlayerEvent.RESUMED){ //btnPlay.setText("||"); btnPlay.setIcon(pauseIcon); } } @Override public void setController(BasicController arg0) {} @Override public void progress(int bytesread, long microseconds, byte[] pcmdata, Map properties) { //we don't want to use microseconds directly because it gets resetted on seeking seekbar.updateSeekBar(player.getProgressMicroseconds(), currentAudioDurationSec); if(wff != null) wff.updateWave(pcmdata); if(fdf != null) fdf.updateWave(pcmdata); } @Override public void opened(Object arg0, Map arg1) { //btnPlay.setText("||"); btnPlay.setIcon(pauseIcon); jSongList.setSelectedIndex(player.getIndexSong()); lblplaying.setText("Now Playing: " + songList.get(player.getIndexSong())); currentAudioDurationSec = player.getAudioDurationSeconds(); } }); //Timers Executor / Every 1 Second timersExec.scheduleAtFixedRate(new Runnable() { @Override public void run() { updateTimers(); //updatePlayingText(); } }, 0, 1, TimeUnit.SECONDS); titleExec.scheduleAtFixedRate(new Runnable() { @Override public void run() { updatePlayingText(); } }, 0, 1, TimeUnit.SECONDS); //Btn Waveform btnShWf.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent arg0) { SwingUtilities.invokeLater(new Runnable(){ @Override public void run() { wff = new WaveformFrame(); wff.setVisible(true); } }); } }); //Btn that show freq diagram btnShDi.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent arg0) { fdf = new FreqDiagFrame(); fdf.setVisible(true); } }); //Open status window frame btnShSt.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent arg0) { //stf = new StatusFrame(); stf.setVisible(true); } }); } /** * Used by the Play/Pause button */ private void tooglePlay() throws BasicPlayerException { if(songList.size() == 0) return; if(!player.isPaused()){ player.pause(); //btnPlay.setText(">"); btnPlay.setIcon(playIcon); } else{player.play();} }

It's a lot of code so let's analyze it in order:

• File chooser: It's a swing component that can be used to create an interface to select one or multiple files. I've set a file filter to let it see only files ending with .mp3 or .wav.
• btnAdd will use the File chooser to get the songs the user selected, adding it to the playlist.
• Song list: contains visual list of the playlist, I've added a behavior so that when a double click occurs it gets the index clicked and plays the selected song.
• Btn Delete: delete the selected songs from the playlist
• Play Btn: a simple play/pause toogle button.
• Btn Next/ Btn Prev: move to next or previous song.
• Player related behaviour: this is where all the fun things happens. We add a new event listener to the basic player, this way we can have some things done every time a certain event happens regarding the player.
• We use progress to update the seekbar
• We use opened to detect if a new song start playing ( a new audio file has been opened by the BasicPlayer class...)
• With state update we detect if the streaming has been paused, stopped, resumed, etc.
• Other: something regarding additional features

There are some concept that we already explained in the previous posts, the code shouldn't be that hard but for problems or anything just let me know in the comments.

Creating the AudioPlayer class for playing music in java

Now that our User Interface it's almost done let's focus on making a class that implements all our player functionality, we'll call this class AudioPlayer. Basically, we are gonna crate a wrapper around a BasicPlayer object, if you don't know basicplayer api, check this page.

BasicPlayer layer is the simple player API of jlGui. These classes are designed to be used in any application that needs simple features (play, stop, pause, resume, seek) to play audio file or stream. It's a high-level API over JavaSound API. 

First off, let's start by defining our class properties, so that our goals are clear, we want to have access to most of this kind of information

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17

//Extension of BasicPlayer private static AudioPlayer instance = null; private ArrayList<String> playlist = new ArrayList<String>(); private int index = 0; private boolean paused = true; private boolean opened = false; private boolean isSeeking = false; //Current Audio Properties private float audioDurationInSeconds = 0; private int audioFrameSize = 0; private float audioFrameRate = 0; //Stream info/status private byte[] cpcmdata; private long csms = 0; //Current Song microseconds private int lastSeekMs = 0; //Every time we seek, basic player returns microseconds are resetted //we need a var to mantain the position we seeked to

Now let's create the constructor, the class it's an extension from BasicPlayer class, since I'm wrapping it. I've used a singleton here because it's pattern sweets the AudioPlayer propose, we'll need and control only one AudioPlayer object.

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59

public class AudioPlayer extends BasicPlayer{ //Want to use a singleton private AudioPlayer() { super(); //Wanna give the AudioPlayer class a basic behaviour this.addBasicPlayerListener(new BasicPlayerListener() { @Override public void stateUpdated(BasicPlayerEvent event) { if(event.getCode() == BasicPlayerEvent.EOM) { lastSeekMs = 0; paused = true; //reset player state opened = false; log("EOM event catched, player resetted."); } if(event.getCode() == BasicPlayerEvent.SEEKING) isSeeking = true; if(event.getCode() == BasicPlayerEvent.SEEKED) isSeeking = false; } @Override public void setController(BasicController arg0) { //No need to use this } @Override public void progress(int bytesread, long microseconds, byte[] pcmdata, Map properties) { csms = microseconds; cpcmdata = pcmdata; } @Override public void opened(Object stream, Map properties) { log("Opened event caught"); Object[] e = properties.entrySet().toArray(); Object[] k = properties.keySet().toArray(); String line = "Stream properties:"; for(int i = 0; i<properties.size(); i++){ line += "\n\t" + k[i] + ":" + e[i]; } log(line); //Set Audio Properties File file = new File(playlist.get(index)); long audioFileLength = file.length(); int frameSize = (int) properties.get("mp3.framesize.bytes"); float frameRate = (float) properties.get("mp3.framerate.fps"); audioFrameSize = frameSize; audioFrameRate = frameRate; audioDurationInSeconds = (audioFileLength / (frameSize * frameRate)); log("\tframesize " + frameSize + " framerate " + frameRate); log("\tAudio File lenght in seconds is: " + audioDurationInSeconds); } }); } public static AudioPlayer getInstance(){ if(instance == null) instance = new AudioPlayer(); return instance; } ///////////////////////////////////// }

While creating the object, I gave the player some of it's basic functionality, by adding to him a behaviour, I did this by adding a BasicListener with this.addBasicPlayerListener. Function in the listener are automatically called every one in a while by the BasicPlayer class (callback functions). Now follows the other implemented methods.

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121

@Override public void play() throws BasicPlayerException { if(playlist.size() == 0) return; if(!paused || !opened){ File f = new File(playlist.get(index)); log("Opening file... " + f.getAbsolutePath()); open(f); opened = true; super.play(); } if(paused) super.resume(); paused = false; } @Override public void pause() throws BasicPlayerException { log("Paused"); paused = true; super.pause(); } @Override public void stop() throws BasicPlayerException { paused = false; super.stop(); } public boolean isPaused(){ return paused; } public boolean isOpenFile() { return opened; } public ArrayList<String> getPlaylist(){ return playlist; } public int getIndexSong(){ return index; } public void setIndexSong(int index){ this.index = index; lastSeekMs = 0; } public boolean isSeeking() { return isSeeking; } /** * goes to the next song in playlist and plays it * @throws BasicPlayerException */ public void nextSong() throws BasicPlayerException{ if(playlist.size() == 0) return; lastSeekMs = 0; paused = false; index = (index+1)%playlist.size(); play(); } /** * goes to the previous song and plays it * @throws BasicPlayerException */ public void prvSong() throws BasicPlayerException{ if(playlist.size() == 0) return; lastSeekMs = 0; paused = false; index = (index-1)%playlist.size(); play(); } /** * Adds a song to the playlist * @param songPath */ public void addSong(String songPath) { playlist.add(songPath); } /** * Remove a song by index * @param index */ public void removeSong(int index) { playlist.remove(index); } /** * Remove a song by songPath * @param songPath */ public void removeSong(String songPath) { playlist.remove(songPath); } public byte[] getPcmData(){return cpcmdata;} public long getProgressMicroseconds(){return csms+lastSeekMs;} public float getAudioDurationSeconds() {return audioDurationInSeconds;} public float getAudioFrameRate() { return audioFrameRate; } public float getAudioFrameSize() { return audioFrameSize; } /** * Remembers what's the last position relative to the playing song * when seeking */ public void setLastSeekPositionInMs(int seekMs) { lastSeekMs = seekMs; } /** * For logging * @param line */ private void log(String line) { System.out.println("AudioPlayer] " + line); MainView.stf.addText("AudioPlayer] " + line); } //End of class

As you have already noticed the code it's not that complex since most of the work is done by BasicPlayer, one of the BasicPlayer class though is that after seeking the returned progress in microsecond of the song is reset, as you can see in the code, this problem has been fixed with the use of one variable.