Обновите код, как показано ниже, чтобы уменьшить область поиска:
#counter {
position: absolute;
width: 300px;
height: 235px;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
text-align: center;
font-family: "Josefin Sans", sans serif;
border: black solid 1px;
transition: 0.3s;
}
.btn_high,
.btn_low {
font-size: 18px;
color: #000;
border: none;
margin: 10px 20px; /* changed */
font-family: "Josefin Sans", sans serif;
background: none;
outline: none;
transition: 0.3s;
cursor: pointer;
}
.btn_high:hover {
color: #4CAF50;
}
.btn_high:active {
color: #8BC34A;
}
.btn_low:hover {
color: #f44336;
}
.btn_low:active {
color: #E91E63;
}
/*
(when you hover besides the number the hover effect works)
*/
#number {
font-size: 40px;
transition: 0.3s;
margin: 55px auto 20px; /*changed*/
display:table; /*added */
}
#number:hover {
color: #607D8B;
}
<div id="counter">
<div id="number">0</div>
<button class="btn_high">Higher</button>
<button class="btn_low">Lower</button>
</div>