Der Name des default branch ist ab jetzt für neue Repositories main. The name of the default branch for new repositories is from now on main.

Commit 8bdc3fe0 authored by Michael Kingston's avatar Michael Kingston Committed by Daniel Müller
Browse files

Fix more image display

parent 0e4524a9
......@@ -26,7 +26,6 @@ different text to speech synthesizers.
* `/recognized_speech` Shows recognized speech in the bottom of the face to support humans in interacting with the robot.
* `/robot_face/ImageDisplay` An Image can be displayed optionally (this topic awaits a image as message).
* `/robot_face/ImageFileDisplay` Alternatively a filename can be specified.
* `/robot_face/expected_input` An additional text on the bottom of the robot face will be shown. Here you can display instructions depending on the state.
* `/robot_face/talking_finished` When coupled with a TTS system this topic will be sent once the robot finished speaking.
* `/robot_face/text_out` When coupled with a TTS system this topic will be used for synthesizing speech.
......
......@@ -59,15 +59,13 @@ public:
/** @brief callback for images from harddisk*/
public slots:
/** @brief hide the current image and stop the reset timer*/
void clearImage();
/** @brief show the current image for time of milliseconds. If there is no image to show, the
* window would be black*/
void showImage(const unsigned int milliseconds);
void pop_image_from_queue();
void processImageQueue();
signals:
void imageProcessed();
void newImageToShow(const unsigned int milliseconds);
void newImageToShow();
void pauseTalkingHead(int state);
protected:
void paintEvent(QPaintEvent*);
......@@ -76,21 +74,16 @@ private:
/** @brief queue to buffer images (necessary if the frequency of incoming images is higher than
* the display frequency)*/
std::queue<std::unique_ptr<QImage>> image_queue_;
std::queue<int> timeout_queue_;
/** @brief timer to specify the duration of painting the current image*/
QTimer* reset_timer_; // no smart pointer when using parenting of Qt
const unsigned int max_image_width_; // in px
const unsigned int max_image_height_; // in px
int window_rotation_; // in degrees and only 0, 90, 180 and 270 are allowed
/** @brief pop the front element in queue and call pointers objects destructor*/
void pop_image_from_queue();
/** @brief rotate and draw the current image. Rotation is respectively to window_rotation_*/
void rotateAndDrawImage(QPainter* painter, const QImage* toDraw);
/** @brief scale image respectively to max_image_width_ or max_image_height_ and add it to the
* image queue*/
//void scaleAndAddImageToQueue(const QImage& image);
/** @brief convert a cv::Mat to a QImage. Supported encodings are CV_8UC4, CV_8UC3 and CV_8UC1*/
QImage cvMatToQImage(const cv::Mat& inMat);
/** @brief actual executive process of image data for painting in the window*/
......
......@@ -100,7 +100,6 @@ private:
ros::Subscriber talking_finished_subscriber_;
ros::Subscriber image_stream_subscriber_;
ros::Subscriber image_stream_show_subscriber_;
ros::Subscriber face_image_stream_show_subscriber_;
ros::Subscriber image_subscriber_;
ros::Subscriber image_show_subscriber_;
......
......@@ -48,10 +48,10 @@
#include <iostream>
#include <istream>
#include <map>
#include <memory>
#include <string>
#include <vector>
#include <utility>
#include <memory>
#include <vector>
#include <cv_bridge/cv_bridge.h>
#include <opencv2/imgproc/imgproc.hpp>
......@@ -60,6 +60,7 @@
#include <homer_robot_face/TextProcessor.h>
#include <homer_robot_face/ImageDisplay.h>
//#include "MainWindow.h" //needed for checking on processed images for visualize stream
......@@ -85,7 +86,7 @@ public:
/** QWidget Constructor */
TalkingHead(QWidget* parent = 0, std::string mesh_string = "",
std::vector<std::vector<float>> material_vector = std::vector<std::vector<float>>(),
int window_rotation = 0);
int window_rotation = 0, std::shared_ptr<ImageDisplay> image_display=nullptr);
/** Destructor */
virtual ~TalkingHead();
......@@ -99,20 +100,13 @@ public:
void callbackVisemes();
void callbackTextForEmotion(const std_msgs::String::ConstPtr& msg);
void callbackResetAnimation(const std_msgs::String::ConstPtr& msg);
void callbackShowStream(const homer_robot_face::DisplayImage::ConstPtr& image_msg);
void callbackShowMatImage(const sensor_msgs::ImageConstPtr& msg);
public slots:
void updateOverlay();
void clearFace();
void setTimer(int msec);
void pausedByImageDisplay(int msec);
signals:
void faceCleared();
void timerChanged(int msec);
protected:
......@@ -121,6 +115,9 @@ protected:
virtual void moveEvent(QMoveEvent* event);
private:
// ref to imagedisplay since pausing will be triggerd by a signal from it
std::shared_ptr<ImageDisplay> image_display_;
/// File Access
std::vector<Ogre::String> phones_;
std::vector<Ogre::String> words_;
......@@ -161,9 +158,11 @@ private:
bool is_visible_;
unsigned int show_time_;
bool init_;
QSize head_size_;
/// Emotion related
std::vector<std::string> smileys_;
const std::vector<std::string> smileys_ = { ">:", ":O", ":o", ":!", ":)", ":(", ".", ":&" };
bool angry_;
bool smile_;
bool sad_;
......@@ -172,6 +171,9 @@ private:
bool disgusted_;
bool afraid_;
void resetEmotions();
void activateEmotion(const std::string& emotion);
/// Ogre::FrameListener
virtual bool frameRenderingQueued(const Ogre::FrameEvent& evt);
......
......@@ -36,8 +36,8 @@ ImageDisplay::ImageDisplay(const int window_rotation, const unsigned int max_wid
this->setVisible(false);
reset_timer_->setSingleShot(true);
connect(reset_timer_, SIGNAL(timeout()), SLOT(clearImage()));
connect(this, SIGNAL(newImageToShow(unsigned int)), SLOT(showImage(unsigned int)));
connect(reset_timer_, SIGNAL(timeout()), SLOT(pop_image_from_queue()));
connect(this, SIGNAL(newImageToShow()), SLOT(processImageQueue()));
}
ImageDisplay::~ImageDisplay()
......@@ -46,49 +46,52 @@ ImageDisplay::~ImageDisplay()
{
pop_image_from_queue();
}
// note: reset_timer will be deleted by parent
if (reset_timer_)
{
delete reset_timer_;
}
}
void ImageDisplay::pop_image_from_queue()
{
if (image_queue_.size()>0)
{
image_queue_.pop();
}
if (timeout_queue_.size()>0)
{
timeout_queue_.pop();
}
emit newImageToShow();
}
void ImageDisplay::showImage(const unsigned int milliseconds)
{
// ROS_INFO("show it to me");
this->show();
update();
reset_timer_->start(milliseconds);
}
void ImageDisplay::clearImage()
void ImageDisplay::processImageQueue()
{
// ROS_INFO("go sleep again");
this->hide();
pop_image_from_queue();
if (image_queue_.empty())
{
this->hide();
emit pauseTalkingHead(0);
}
else
{
emit pauseTalkingHead(timeout_queue_.front());
this->show();
update();
}
}
void ImageDisplay::paintEvent(QPaintEvent*)
{
QPainter qpainter(this);
switch (image_queue_.size())
if (image_queue_.size() == 0)
{
case 0:
// no image to paint
return;
case 1:
// if there is only one element in the queue we do not remove it here, because we have to
// wait until the timer fires;
rotateAndDrawImage(&qpainter, image_queue_.front().get());
break;
default:
// more than one image in the queue, paint the image at the front of the queue and pop it
// then
reset_timer_->start(0);
}
else
{
reset_timer_->start(timeout_queue_.front());
rotateAndDrawImage(&qpainter, image_queue_.front().get());
pop_image_from_queue();
break;
}
}
......@@ -190,15 +193,8 @@ void ImageDisplay::processImageData(const homer_robot_face::DisplayImage::ConstP
// and from OpenCV to QImage
auto imageQ = std::make_unique<QImage>(cvMatToQImage(sensor_img_cv->image));
// scale image to the greater value of height or width to fit in widget
std::unique_ptr<QImage> image_ptr;
if (imageQ->height() > imageQ->width())
{
image_ptr = std::make_unique<QImage>(imageQ->scaledToHeight(max_image_height_));
}
else
{
image_ptr = std::make_unique<QImage>(imageQ->scaledToWidth(max_image_width_));
}
std::unique_ptr<QImage> image_ptr = std::make_unique<QImage>(
imageQ->scaled(QSize(max_image_width_, max_image_height_), Qt::KeepAspectRatio));
// forcing a deep copy
if (!image_ptr->isDetached())
......@@ -207,14 +203,18 @@ void ImageDisplay::processImageData(const homer_robot_face::DisplayImage::ConstP
}
image_queue_.push(std::move(image_ptr));
timeout_queue_.push(image_msg->time * 1000);
}
void ImageDisplay::callbackImageDisplay(const homer_robot_face::DisplayImage::ConstPtr& image_msg)
{
ROS_INFO_STREAM("Received ImageDisplay msg, draw image for " << image_msg->time << "seconds.");
// ROS_INFO_STREAM("Received ImageDisplay msg, draw image for " << image_msg->time << "seconds.");
processImageData(image_msg);
emit newImageToShow(image_msg->time * 1000); // convert milliseconds to seconds
if (image_queue_.size()<2)
{
emit newImageToShow();
}
}
......
......@@ -28,12 +28,12 @@ MainWindow::MainWindow(QWidget* parent)
ROS_INFO("Creating MainWindow..");
// TODO FIXME
windowHeight_ = 800;
windowWidth_ = 600;
windowHeight_ = 1000; //1000!
windowWidth_ = 800; //800!
int width = 600;
int height = 800;
int width = windowWidth_;
int height = windowHeight_;
int window_rotation = 0;
int max_width = width;
......@@ -41,6 +41,14 @@ MainWindow::MainWindow(QWidget* parent)
int min_width = width;
int min_height = height;
// ImageStreamDisplay
image_stream_display_
= std::make_shared<ImageDisplay>(window_rotation, windowWidth_, windowHeight_, this);
//image_stream_display_->setMinimumSize(width, height);
//image_stream_display_->setMaximumSize(min_width, min_height);
try
{
std::vector<std::vector<float>> material_colors;
......@@ -57,7 +65,7 @@ MainWindow::MainWindow(QWidget* parent)
window_rotation = cfg.get("Window Rotation");
assert(window_rotation == 0 || window_rotation == 90 || window_rotation == 180
|| window_rotation == 270);
talking_head_ = std::make_unique<TalkingHead>(this, mesh_filename, material_colors, window_rotation);
talking_head_ = std::make_unique<TalkingHead>(this, mesh_filename, material_colors, window_rotation, image_stream_display_);
std::cout << "Mesh Filename : " << mesh_filename << std::endl;
}
......@@ -66,38 +74,35 @@ MainWindow::MainWindow(QWidget* parent)
std::cerr << e.what() << std::endl;
}
if (window_rotation == 0 || window_rotation == 180)
{
center_layout = new QVBoxLayout();
main_layout = new QVBoxLayout();
h_layout = new QVBoxLayout();
setMinimumSize(width, height);
max_width = width / 4;
max_height = height / 4;
min_width = (width * 3) / 4;
min_height = (height * 3) / 4;
height = max_height;
min_width = width;
if (window_rotation == 0 || window_rotation == 180)
{
setMinimumSize(width, height);
}
else // if( window_rotation == 90 || window_rotation == 270 )
{
center_layout = new QHBoxLayout();
main_layout = new QHBoxLayout();
h_layout = new QHBoxLayout();
int tmp = height;
auto tmp = height;
height = width;
width = tmp;
setMinimumSize(width, height);
}
max_width = width / 4;
max_height = height / 4;
min_width = (width * 3) / 4;
min_height = (height * 3) / 4;
if (window_rotation == 0 || window_rotation == 180)
{
height = max_height;
min_width = width;
}
else // if( window_rotation == 90 || window_rotation == 270 )
{
width = max_width;
min_height = height;
}
......@@ -111,7 +116,7 @@ MainWindow::MainWindow(QWidget* parent)
setAutoFillBackground(true);
// Center Layout (fixed size to fit small screen)
QWidget* center_widget = new QWidget();
center_widget = new QWidget();
center_widget->setContentsMargins(0, 0, 0, 0);
center_layout->setSpacing(0);
......@@ -120,12 +125,6 @@ MainWindow::MainWindow(QWidget* parent)
#if USE_FESTIVAL
festival_generator_ = std::make_shared<FestivalGenerator>();
#endif
// ImageStreamDisplay
image_stream_display_
= std::make_shared<ImageDisplay>(window_rotation, windowWidth_, windowHeight_ - 60, this);
image_stream_display_->setMinimumSize(width, height - 60);
image_stream_display_->setMaximumSize(min_width, min_height - 60);
// ExpectedInputDisplay
expc_inp_display_ = std::make_shared<TextOutDisplay>(0, 20, ::EXP, window_rotation, this);
......@@ -143,35 +142,39 @@ MainWindow::MainWindow(QWidget* parent)
text_rec_display_->setPalette(custom_palette);
text_rec_display_->setMaximumSize(width, height);
// Main Layout
// Layout ------------------------------------------------
main_layout->setSpacing(0);
main_layout->setContentsMargins(0, 0, 0, 0);
if (window_rotation == 0 || window_rotation == 270)
{
center_layout->addWidget(expc_inp_display_.get());
center_layout->addStretch();
center_layout->addWidget(text_out_display_.get());
center_layout->addStretch();
center_layout->addWidget(text_rec_display_.get());
center_layout->addWidget(expc_inp_display_.get());
center_layout->addStretch();
center_layout->addWidget(text_out_display_.get());
center_layout->addStretch();
center_layout->addWidget(text_rec_display_.get());
center_layout->setStretchFactor(text_out_display_.get(), 1);
center_layout->setStretchFactor(text_out_display_.get(), 1);
// center_widget->setMinimumSize( width, height );
center_widget->setMaximumSize(width, height);
center_widget->setLayout(center_layout);
// center_widget->setMinimumSize( width, height );
center_widget->setMaximumSize(width, height);
center_widget->setLayout(center_layout);
h_layout->setSpacing(0);
h_layout->setContentsMargins(0, 0, 0, 0);
talking_head_->setMinimumSize(width, height);
talking_head_->setMaximumSize(min_width, min_height);
// This crap is sensitive to what order you put things so basically impossible to understand/clean up
if (window_rotation == 0 || window_rotation == 270)
{
main_layout->addWidget(talking_head_.get());
talking_head_->setMinimumSize(width, height);
talking_head_->setMaximumSize(min_width, min_height);
main_layout->setStretchFactor(talking_head_.get(), 1);
main_layout->addWidget(image_stream_display_.get()); // comment in
main_layout->setStretchFactor(image_stream_display_.get(), 1); // comment in
h_layout->setSpacing(0);
h_layout->setContentsMargins(0, 0, 0, 0);
h_layout->addStretch();
h_layout->addWidget(center_widget);
......@@ -180,25 +183,9 @@ MainWindow::MainWindow(QWidget* parent)
main_layout->addStretch();
main_layout->addLayout(h_layout);
main_layout->addStretch();
std::cout << "end of main_layout stuff" << std::endl;
}
else // if( window_rotation == 90 || window_rotation == 180 )
{
center_layout->addWidget(text_rec_display_.get());
center_layout->addStretch();
center_layout->addWidget(text_out_display_.get());
center_layout->addStretch();
center_layout->addWidget(expc_inp_display_.get());
center_layout->setStretchFactor(text_out_display_.get(), 1);
// center_widget->setMinimumSize( width, height );
center_widget->setMaximumSize(width, height);
center_widget->setLayout(center_layout);
h_layout->setSpacing(0);
h_layout->setContentsMargins(0, 0, 0, 0);
h_layout->addStretch();
h_layout->addWidget(center_widget);
h_layout->addStretch();
......@@ -208,17 +195,16 @@ MainWindow::MainWindow(QWidget* parent)
main_layout->addStretch();
main_layout->addWidget(talking_head_.get());
talking_head_->setMinimumSize(width, height);
talking_head_->setMaximumSize(min_width, min_height);
main_layout->setStretchFactor(talking_head_.get(), 1);
main_layout->addWidget(image_stream_display_.get());
main_layout->setStretchFactor(image_stream_display_.get(), 1);
}
// main_layout->setSizeConstraint(QLayout::SetFixedSize); //TODO: toProve
setLayout(main_layout);
std::cout << "end of main window constructor" << std::endl;
std::ostringstream next_command;
next_command << "Info: " << qApp->desktop()->screenNumber(talking_head_.get()) << endl;
ROS_INFO("%s", next_command.str().c_str());
......
......@@ -55,12 +55,6 @@ void QtRosNode::subscribeTopics()
face_talking_finished_subscriber_
= text_out_node_handle_->subscribe<std_msgs::String>("/robot_face/talking_finished", 10,
&TalkingHead::callbackResetAnimation, main_window_->getFaceWidget().get());
// face_image_stream_show_subscriber_ =
// node_handle_->subscribe<std_msgs::Int8>( "/robot_face/image_stream",
// 1000, &TalkingHead::callbackShowStream, main_window_->getFaceWidget() );
face_image_stream_show_subscriber_
= node_handle_->subscribe<homer_robot_face::DisplayImage>("/robot_face/ImageDisplay", 10,
&TalkingHead::callbackShowStream, main_window_->getFaceWidget().get());
// text output subscribers
text_out_subscriber_
......
......@@ -23,7 +23,7 @@
TalkingHead::TalkingHead(QWidget* parent, std::string mesh_string,
std::vector<std::vector<float>> material_vector, int window_rotation)
std::vector<std::vector<float>> material_vector, int window_rotation, std::shared_ptr<ImageDisplay> image_display)
: QWidget(parent)
, phones_(0)
, words_(0)
......@@ -60,23 +60,17 @@ TalkingHead::TalkingHead(QWidget* parent, std::string mesh_string,
redraw_timer_ = new QTimer(this); // create internal timer
connect(redraw_timer_, SIGNAL(timeout()), SLOT(updateOverlay()));
connect(this, SIGNAL(timerChanged(int)), SLOT(setTimer(int)));
connect(this, SIGNAL(faceCleared()), SLOT(clearFace()));
ROS_INFO("Initialising talkinghead - waiting 3000 ms for things to get ready.");
redraw_timer_->start(3000);
image_display_ = image_display;
connect(image_display_.get(), SIGNAL(pauseTalkingHead(int)), SLOT(pausedByImageDisplay(int)));
ROS_INFO("Initialising talkinghead - waiting 1000 ms for things to get ready.");
redraw_timer_->start(1000);
redraw_timer_->setSingleShot(true);
initPhoneMap();
srand((unsigned)time(0));
smileys_.push_back(">:");
smileys_.push_back(":O");
smileys_.push_back(":o");
smileys_.push_back(":!");
smileys_.push_back(":)");
smileys_.push_back(":(");
smileys_.push_back(".");
smileys_.push_back(":&");
mesh_string_ = mesh_string;
material_vector_ = material_vector;
......@@ -105,6 +99,41 @@ TalkingHead::~TalkingHead()
}
//---------------------------------------------------------------------------
void TalkingHead::setTimer(int msec)
{
redraw_timer_->start(msec);
}
void TalkingHead::pausedByImageDisplay(int msec)
{
if (msec == 0)
{
window_->setHidden(false);
if (head_size_.isValid())
{
setMinimumSize(800, 250);//(head_size_.width(), head_size_.height() / 3);
setMaximumSize(800, 800);//(head_size_.width(), head_size_.height()); //somehow head shrinks...
}
emit timerChanged(0);
}
else
{
redraw_timer_->stop();
head_size_ = size();
setMinimumSize(1, 1);
setMaximumSize(1, 1);
// Do not set the qwidget to hidden - this will break the ogre renderwindow
// this->setVisible(false);
// Instead hide the Ogre renderwindow
window_->setHidden(true);
}
}
void TalkingHead::updateOverlay()
{
// Restart the timer
......@@ -117,9 +146,9 @@ void TalkingHead::updateOverlay()
initOgreSystem();
}
is_visible_ = true;
window_->setHidden(!is_visible_);
this->setVisible(is_visible_);
// is_visible_ = true;
// window_->setHidden(!is_visible_);
// this->setVisible(is_visible_);
// Render next frame
if (!root_->renderOneFrame())
......@@ -137,7 +166,7 @@ void TalkingHead::updateOverlay()
window_->update();
update();
//Ogre::WindowEventUtilities::messagePump();
// Ogre::WindowEventUtilities::messagePump();
}
//---------------------------------------------------------------------------
......@@ -643,6 +672,27 @@ std::map<float, Ogre::String> TalkingHead::createWordMap()
return words;
}
//---------------------------------------------------------------------------
void TalkingHead::resetEmotions()
{
angry_ = smile_ = sad_ = surprised_ = disgusted_ = afraid_ = false;
rest_ = true;
}
void TalkingHead::activateEmotion(const std::string& emotion)
{
resetEmotions();
afraid_ = emotion.compare("afraid") == 0;
angry_ = emotion.compare("angry") == 0;
disgusted_ = emotion.compare("disgusted") == 0;
sad_ = emotion.compare("sad") == 0;
smile_ = emotion.compare("smile") == 0;
surprised_ = emotion.compare("surprised") == 0;
if (afraid_ || angry_ || disgusted_ || sad_ || smile_ || surprised_)
rest_ = false;
}
void TalkingHead::playTalkAnimation()
{
......@@ -665,6 +715,7 @@ void TalkingHead::playTalkAnimation()