Pitch Bending Directly in the Piano Roll (#7759)

This PR modifies how the Detuning tool works in the piano roll to let the user modify the automation curve right on the notes. This is actually fairly simple, as automation clips already contain the functions for dragging and removing points like in the Automation Editor. So essentially all that's being done is calculating the relative pos of the mouse to the nearest note, and setting the drag value in the automation clip to that position and key.

You can still access the old automation editor version by shift-click

messmerd asked if the code could be made general for any future note parameter besides detuning, in preparation for CLAP per-note parameter automation. I have refactored the relevant functions to accept an enum type for the kind of note parameter, although currently only detuning is supported.

---------

Co-authored-by: Sotonye Atemie <sakertooth@gmail.com>
Co-authored-by: Sotonye Atemie <satemiej@gmail.com>
Co-authored-by: szeli1 <143485814+szeli1@users.noreply.github.com>

include/Note.h

@@ -41,6 +41,7 @@ namespace lmms
 
 
 class DetuningHelper;
+class AutomationClip;
 
 
 enum class Key : int
@@ -123,6 +124,13 @@ public:
 	Type type() const { return m_type; }
 	inline void setType(Type t) { m_type = t; }
 
+	//! Types of per-note automation. Currently only detuning/pitch bending is supported.
+	enum class ParameterType
+	{
+		Detuning = 0
+	};
+	AutomationClip* parameterCurve(ParameterType paramType);
+
 	// used by GUI
 	inline void setSelected( const bool selected ) { m_selected = selected; }
 	inline void setOldKey( const int oldKey ) { m_oldKey = oldKey; }

include/PianoRoll.h

@@ -456,6 +456,8 @@ private:
 	void drawDetuningInfo( QPainter & _p, const Note * _n, int _x, int _y ) const;
 	bool mouseOverNote();
 	Note * noteUnderMouse();
+	//! Calculates the closest note to the mouse given their parameter automation curve
+	Note* parameterEditNoteUnderMouse(Note::ParameterType paramType);
 
 	// turn a selection rectangle into selected notes
 	void computeSelectedNotes( bool shift );
@@ -473,6 +475,20 @@ private:
 
 	void updateKnifePos(QMouseEvent* me, bool initial);
 
+	//! Varaibles which hold which mouse buttons are being held while editing the detuning/parameter of notes.
+	bool m_parameterEditDownLeft = false;
+	bool m_parameterEditDownRight = false;
+	//! Stores the last edited position for the note detuning/parameter curves.
+	//! When erasing nodes when dragging the mouse, all nodes in the range of the last mouse pos to the current mouse pos are removed. Without this, when dragging the mouse super fast, some nodes could get missed; this ensures all nodes from the previous mouse position to the current one will get deleted.
+	std::optional<int> m_lastParameterEditTick = std::nullopt;
+	//! The current note whose detuning/parameter curve is being edited.
+	Note* m_parameterEditClickedNote;
+
+	//! Updates the currently dragged node position in the detuning/parameter curves of the selected notes.
+	void updateParameterEditPos(QMouseEvent* me, Note::ParameterType paramType);
+	//! Finishes the dragging of the current node of the detuning/parameter curves
+	void applyParameterEditPos(Note::ParameterType paramType);
+
 	//! Stores the chords for the strum tool
 	std::vector<NoteVector> m_selectedChords;
 	//! Computes which notes belong to which chords from the selection

src/core/Note.cpp

@@ -224,7 +224,17 @@ void Note::loadSettings( const QDomElement & _this )
 }
 
 
-
+AutomationClip* Note::parameterCurve(ParameterType paramType)
+{
+	switch (paramType)
+	{
+	// TODO: In the future, more per-note parameters which can be automated are planned. Perhaps velocity, panning, and/or some other plugin-specific parameter.
+	case ParameterType::Detuning:
+		return detuning()->automationClip();
+	default:
+		return nullptr;
+	}
+}
 
 
 void Note::createDetuning()

src/gui/editors/PianoRoll.cpp

@@ -118,6 +118,9 @@ const int INITIAL_START_KEY = Octave::Octave_4 + Key::C;
 const int NUM_EVEN_LENGTHS = 6;
 const int NUM_TRIPLET_LENGTHS = 5;
 
+// Radius of the automation node circles which appear when pitchbending a note
+const int DETUNING_HANDLE_RADIUS = 3;
+
 SimpleTextFloat * PianoRoll::s_textFloat = nullptr;
 
 static std::array<QString, 12> s_noteStrings {
@@ -1082,6 +1085,7 @@ void PianoRoll::drawDetuningInfo( QPainter & _p, const Note * _n, int _x,
 								int _y ) const
 {
 	int middle_y = _y + m_keyLineHeight / 2;
+	_p.setBrush(QBrush(m_noteColor));
 	_p.setPen(m_noteColor);
 	_p.setClipRect(
 		m_whiteKeyWidth,
@@ -1158,6 +1162,22 @@ void PianoRoll::drawDetuningInfo( QPainter & _p, const Note * _n, int _x,
 		old_x = cur_x;
 		old_y = cur_y;
 	}
+
+	if (m_editMode == EditMode::Detuning && _n->selected())
+	{
+		for (timeMap::const_iterator it = map.begin(); it != map.end(); ++it)
+		{
+			int curTicks = POS(it);
+			int curX = _x + curTicks * m_ppb / TimePos::ticksPerBar();
+			const float curLevel = INVAL(it);
+			int curY = middle_y - curLevel * m_keyLineHeight;
+			_p.drawEllipse(
+				curX - DETUNING_HANDLE_RADIUS,
+				curY - DETUNING_HANDLE_RADIUS,
+				2 * DETUNING_HANDLE_RADIUS,
+				2 * DETUNING_HANDLE_RADIUS);
+		}
+	}
 }
 
 
@@ -1656,22 +1676,47 @@ void PianoRoll::mousePressEvent(QMouseEvent * me )
 		return;
 	}
 
-	if( m_editMode == EditMode::Detuning && noteUnderMouse() )
+	if (m_editMode == EditMode::Detuning)
 	{
-		static QPointer<AutomationClip> detuningClip = nullptr;
-		if (detuningClip.data() != nullptr)
+		// Let users access automation editor with shift-click, if they want the old functionality
+		Note* clickedNote = noteUnderMouse();
+		if (clickedNote && me->modifiers() & Qt::ShiftModifier)
 		{
-			detuningClip->disconnect(this);
+			if (clickedNote->detuning() == nullptr)
+			{
+				clickedNote->createDetuning();
+				AutomationClip* detuningClip = clickedNote->detuning()->automationClip();
+				connect(detuningClip, SIGNAL(dataChanged()), this, SLOT(update()));
+			}
+			getGUI()->automationEditor()->setGhostMidiClip(m_midiClip);
+			getGUI()->automationEditor()->open(clickedNote->detuning()->automationClip());
+			return;
 		}
-		Note* n = noteUnderMouse();
-		if (n->detuning() == nullptr)
+
+		// Only the currently selected notes are used for detuning
+		// If the user is clicking on a note without any others selected, make that note selected so that it will be counted
+		if (clickedNote && !clickedNote->selected() && getSelectedNotes().empty())
 		{
-			n->createDetuning();
+			clickedNote->setSelected(true);
+			// If there are already automation nodes in the detuning curve, don't immediately place a new when the user clicks on the note to select it
+			if (clickedNote->detuning() && clickedNote->detuning()->hasAutomation()) { return; }
 		}
-		detuningClip = n->detuning()->automationClip();
-		connect(detuningClip.data(), SIGNAL(dataChanged()), this, SLOT(update()));
-		getGUI()->automationEditor()->setGhostMidiClip(m_midiClip);
-		getGUI()->automationEditor()->open(detuningClip);
+
+		// Create detuning curves for each note if they don't have them already
+		for (Note* note: getSelectedNotes())
+		{
+			if (note->detuning() == nullptr)
+			{
+				note->createDetuning();
+				AutomationClip* detuningClip = note->detuning()->automationClip();
+				connect(detuningClip, SIGNAL(dataChanged()), this, SLOT(update()));
+			}
+		}
+
+		m_midiClip->addJournalCheckPoint();
+
+		// Perform the dragging/adding/removing of automation nodes
+		updateParameterEditPos(me, Note::ParameterType::Detuning);
 		return;
 	}
 
@@ -2357,6 +2402,11 @@ void PianoRoll::mouseReleaseEvent( QMouseEvent * me )
 		}
 	}
 
+	if (m_editMode == EditMode::Detuning || m_ctrlMode == EditMode::Detuning)
+	{
+		applyParameterEditPos(Note::ParameterType::Detuning);
+	}
+
 	if( me->button() & Qt::RightButton )
 	{
 		m_mouseDownRight = false;
@@ -2452,7 +2502,11 @@ void PianoRoll::mouseMoveEvent( QMouseEvent * me )
 	{
 		updateKnifePos(me, false);
 	}
-
+	if (m_editMode == EditMode::Detuning && (m_parameterEditDownLeft || m_parameterEditDownRight))
+	{
+		// Update the current dragging/adding/removal of automation nodes in the detuning curves of the selected notes.
+		updateParameterEditPos(me, Note::ParameterType::Detuning);
+	}
 	// Update Strum position if we are on knife mode
 	if (m_editMode == EditMode::Strum && m_strumEnabled)
 	{
@@ -2757,6 +2811,105 @@ void PianoRoll::mouseMoveEvent( QMouseEvent * me )
 }
 
 
+void PianoRoll::updateParameterEditPos(QMouseEvent* me, Note::ParameterType paramType)
+{
+	// If this is the first time this function is called (not mouseMove or mouseRelease), initialize the variables.
+	if (me->type() == QEvent::MouseButtonPress)
+	{
+		// Mouse button press event does not seem to remember buttons which are already pressed, so the || keeps the previous state.
+		m_parameterEditDownLeft = m_parameterEditDownLeft || me->button() & Qt::LeftButton;
+		// Do not allow right click mode if left click is already held.
+		m_parameterEditDownRight = (m_parameterEditDownRight || me->button() & Qt::RightButton) && m_parameterEditDownLeft == false;
+		m_lastParameterEditTick = std::nullopt;
+		// Get the note with the closest automation curve to the mouse cursor
+		m_parameterEditClickedNote = parameterEditNoteUnderMouse(paramType);
+	}
+
+	if (!m_parameterEditClickedNote) { return; }
+
+	// Calculate the key and time of the mouse cursor in the piano roll
+	int keyNum = getKey(me->y());
+	int posTicks = (me->x() - m_whiteKeyWidth) *
+			TimePos::ticksPerBar() / m_ppb + m_currentPosition;
+
+	// Calculate the relative position of the mouse with respect to the note.
+	TimePos relativePos = posTicks - m_parameterEditClickedNote->pos();
+	int relativeKey = keyNum - m_parameterEditClickedNote->key();
+
+	// Set the quantization of the automation editor to match the piano roll. This is not an ideal system, but it works.
+	AutomationClip::setQuantization(quantization());
+
+	// Loop through all of the selected notes and update the drag position in each.
+	bool updateLastEditTick = true; // Only update m_lastParameterEditTick if the user isn't trying to drag the first node left/right, or any other node onto the first node.
+	for (Note* note: getSelectedNotes())
+	{
+		AutomationClip* aClip = note->parameterCurve(paramType);
+		if (aClip == nullptr) { continue; }
+		// If left-clicking, add/drag a node.
+		if (m_parameterEditDownLeft)
+		{
+			// Don't allow the user to drag the first node from the start of the note. They can drag it up and down, but if they try to move it from the first tick, apply the previous drag and start a new one to preserve the node
+			if (m_lastParameterEditTick != std::nullopt && Note::quantized(m_lastParameterEditTick.value() - m_parameterEditClickedNote->pos(), quantization()) == 0 && Note::quantized(relativePos, quantization()) != 0)
+			{
+				updateLastEditTick = false;
+				aClip->setDragValue(0, relativeKey);
+			}
+			// Also, don't let the user drag another node onto the first node, since that creates issues with the first node changing height without the user intending it to
+			else if (m_lastParameterEditTick != std::nullopt && Note::quantized(m_lastParameterEditTick.value() - m_parameterEditClickedNote->pos(), quantization()) > 0 && Note::quantized(relativePos, quantization()) <= 0)
+			{
+				updateLastEditTick = false;
+				aClip->setDragValue(quantization(), relativeKey);
+			}
+			else
+			{
+				aClip->setDragValue(relativePos, relativeKey);
+			}
+		}
+		// If right-clicking, remove nodes.
+		else if (m_parameterEditDownRight)
+		{
+			// If the last position of the mouse exists, remove all nodes which the mouse has dragged over since the last mouse event.
+			// std::nullopt means there is no previous mouse position, meaning that the drag just started.
+			if (m_lastParameterEditTick != std::nullopt)
+			{
+				// Don't allow the user to accidentally remove the default node at the very start of the note, by making sure the tick range is >= 1
+				if (!(m_lastParameterEditTick.value() - m_parameterEditClickedNote->pos() == 0 && relativePos == 0))
+				{
+					aClip->removeNodes(std::max(1, m_lastParameterEditTick.value() - m_parameterEditClickedNote->pos()), std::max(TimePos{1}, relativePos));
+				}
+			}
+			else
+			{
+				// If the drag just started, only remove at that one time pos
+				// But don't allow the user to delete the first node
+				if (relativePos != 0)
+				{
+					aClip->removeNode(relativePos);
+				}
+			}
+		}
+	}
+	if (updateLastEditTick) { m_lastParameterEditTick = posTicks; }
+}
+
+void PianoRoll::applyParameterEditPos(Note::ParameterType paramType)
+{
+	// If the left button was just released, apply the drag on all of the notes' automation clips.
+	if (m_parameterEditDownLeft)
+	{
+		for (Note* note: getSelectedNotes())
+		{
+			AutomationClip* aClip = note->parameterCurve(paramType);
+			if (aClip == nullptr) { continue; }
+
+			aClip->applyDragValue();
+		}
+	}
+	m_parameterEditDownRight = false;
+	m_parameterEditDownLeft = false;
+	m_parameterEditClickedNote = nullptr;
+}
+
 
 
 void PianoRoll::updateKnifePos(QMouseEvent* me, bool initial)
@@ -3593,12 +3746,17 @@ void PianoRoll::paintEvent(QPaintEvent * pe )
 			}
 
 			int pos_ticks = note->pos();
-
 			int note_width = len_ticks * m_ppb / TimePos::ticksPerBar();
+
+			int detuningLength = !note->detuning()->automationClip()->getTimeMap().isEmpty()
+				? note->detuning()->automationClip()->getTimeMap().lastKey() * m_ppb / TimePos::ticksPerBar()
+				: note_width;
+
 			const int x = ( pos_ticks - m_currentPosition ) *
 					m_ppb / TimePos::ticksPerBar();
-			// skip this note if not in visible area at all
-			if (!(x + note_width >= 0 && x <= width() - m_whiteKeyWidth))
+			// Skip this note if not in visible area at all
+			// But still draw the note if the detuning curve extends past the end of it.
+			if (!(x + std::max(note_width, detuningLength) >= 0 && x <= width() - m_whiteKeyWidth))
 			{
 				continue;
 			}
@@ -4335,6 +4493,12 @@ void PianoRoll::verScrolled( int new_pos )
 
 void PianoRoll::setEditMode(int mode)
 {
+	if (static_cast<EditMode>(mode) == EditMode::Detuning)
+	{
+		TextFloat::displayMessage(tr("Pitch Bending"),
+			tr("Click and drag on a note or selection to edit its detuning curve\nShift-click to open the note in Automation Editor"),
+			embed::getIconPixmap("automation"), 4000);
+	}
 	m_ctrlMode = m_editMode = (EditMode) mode;
 }
 
@@ -4905,6 +5069,55 @@ Note * PianoRoll::noteUnderMouse()
 	return nullptr;
 }
 
+
+Note * PianoRoll::parameterEditNoteUnderMouse(Note::ParameterType paramType)
+{
+	QPoint pos = mapFromGlobal(QCursor::pos());
+
+	if (pos.x() <= m_whiteKeyWidth
+		|| pos.x() > width() - SCROLLBAR_SIZE
+		|| pos.y() < PR_TOP_MARGIN
+		|| pos.y() > keyAreaBottom())
+	{
+		return nullptr;
+	}
+
+	// Get the key and time pos of the cursor
+	int keyNum = getKey(pos.y());
+	int posTicks = (pos.x() - m_whiteKeyWidth) *
+			TimePos::ticksPerBar() / m_ppb + m_currentPosition;
+
+	// Loop through all notes having their detuning/parameter being edited, and find the one whose automation curve is closest to the mouse.
+	int minScore = -1;
+	Note* closestNote = nullptr;
+	for (Note* note : getSelectedNotes())
+	{
+		// Skip note if the mouse is outside of its start time
+		if (posTicks < note->pos()) { continue; }
+
+		TimePos relativePos = posTicks - note->pos();
+		int relativeKey = keyNum - note->key();
+
+		AutomationClip* aClip = note->parameterCurve(paramType);
+		if (aClip == nullptr) { continue; }
+
+		int differenceFromCurve = std::abs(relativeKey - aClip->valueAt(relativePos));
+		int verticalPixelOffset = differenceFromCurve * m_keyLineHeight;
+
+		int lastNodeTime = !aClip->getTimeMap().isEmpty() ? aClip->getTimeMap().lastKey() : 0;
+		int horizontalPixelOffset = (posTicks > note->endPos() && relativePos > lastNodeTime) ? (relativePos - lastNodeTime) * m_ppb / TimePos::ticksPerBar() : 0;
+
+		int distanceScore = verticalPixelOffset * verticalPixelOffset + horizontalPixelOffset * horizontalPixelOffset;
+
+		if (distanceScore < minScore || minScore == -1)
+		{
+			minScore = distanceScore;
+			closestNote = note;
+		}
+	}
+	return closestNote;
+}
+
 void PianoRoll::changeSnapMode()
 {
 	//	GridMode::Nudge,